Commit 737cd45b authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

preprocess.py: includes: simple test cases completed

parent 05e4860e
......@@ -38,7 +38,7 @@ The central class of module DHParser's ``error`` is the ``Error``-class.
The easiest way to create an error object is by instantiating
the Error class with an error message and a source position::
>>> error = Error('Somethigs went wrong', 123)
>>> error = Error('Something went wrong', 123)
>>> print(error)
Error (1000): Something went wrong
......
......@@ -337,52 +337,104 @@ def generate_include_map(source_name: str,
find_next_include: FindIncludeFunc) -> Tuple[IncludeMap, str]:
file_names: set = set()
# def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
# nonlocal file_names
# map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
# text_chunks: List[str] = []
# source_offset: int = 0
# if source_name in file_names:
# raise ValueError(f'Circular include of {source_name} detected!')
# file_names.add(source_name)
# last_begin = -1
# last_end = 0
# lengths = 0
# begin, length, include_name = find_next(source_text, 0)
# while begin >= 0:
# assert begin > last_begin
# with open(include_name, 'r', encoding='utf-8') as f:
# include_text = f.read()
# inner_map, inner_text = generate_map(include_name, include_text, find_next)
# inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
# inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
# if begin == map.positions[-1]: # FEHLER!
# map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
# map.positions = map.positions[:-1] + inner_map.positions[:-1]
# map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
# text_chunks.append(inner_text)
# else:
# text_chunks.append(source_text[last_end:begin])
# source_offset += begin - last_end
# map.file_names += inner_map.file_names[:-1]
# map.positions += inner_map.positions[:-1]
# map.offsets += inner_map.offsets[:-1]
# text_chunks.append(inner_text)
# lengths += length
# map.file_names.append(source_name)
# map.positions.append(inner_map.positions[-1])
# map.offsets.append(source_offset + lengths - inner_map.positions[-1])
# last_end = begin + length
# last_begin = begin
# begin, length, include_name = find_next(source_text, last_end)
# rest = source_text[last_end:]
# if rest:
# text_chunks.append(rest)
# map.positions.append(map.positions[-1] + len(rest))
# map.offsets.append(map.offsets[-1])
# map.file_names.append(source_name)
# file_names.remove(source_name)
# return map, ''.join(text_chunks)
def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
nonlocal file_names
map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
text_chunks: List[str] = []
source_offset: int = 0
map = IncludeMap(source_name, [0], [0], [source_name])
result = []
if source_name in file_names:
raise ValueError(f'Circular include of {source_name} detected!')
file_names.add(source_name)
source_pointer = 0
source_offset = 0
result_pointer = 0
last_begin = -1
last_end = 0
lengths = 0
begin, length, include_name = find_next(source_text, 0)
while begin >= 0:
assert begin > last_begin
source_delta = begin - source_pointer
source_pointer += source_delta
result_pointer += source_delta
with open(include_name, 'r', encoding='utf-8') as f:
include_text = f.read()
inner_map, inner_text = generate_map(include_name, include_text, find_next)
inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
if begin == map.positions[-1]: # FEHLER!
included_text = f.read()
inner_map, inner_text = generate_map(include_name, included_text, find_next)
inner_map.positions = [pos + result_pointer for pos in inner_map.positions]
inner_map.offsets = [offset - result_pointer for offset in inner_map.offsets]
if source_delta == 0:
map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
map.positions = map.positions[:-1] + inner_map.positions[:-1]
map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
text_chunks.append(inner_text)
result.append(inner_text)
else:
text_chunks.append(source_text[last_end:begin])
source_offset += begin - last_end
result.append(source_text[source_pointer - source_delta: source_pointer])
map.file_names += inner_map.file_names[:-1]
map.positions += inner_map.positions[:-1]
map.offsets += inner_map.offsets[:-1]
text_chunks.append(inner_text)
lengths += length
result.append(inner_text)
inner_length = len(inner_text)
result_pointer += inner_length
map.file_names.append(source_name)
map.positions.append(inner_map.positions[-1])
map.offsets.append(source_offset + lengths - inner_map.positions[-1])
last_end = begin + length
last_begin = begin
begin, length, include_name = find_next(source_text, last_end)
rest = source_text[last_end:]
map.positions.append(result_pointer)
source_pointer += length
source_offset += length - inner_length
map.offsets.append(source_offset)
begin, length, include_name = find_next(source_text, source_pointer)
rest = source_text[source_pointer:]
if rest:
text_chunks.append(rest)
result.append(rest)
map.positions.append(map.positions[-1] + len(rest))
map.offsets.append(map.offsets[-1])
map.offsets.append(source_offset)
map.file_names.append(source_name)
file_names.remove(source_name)
return map, ''.join(text_chunks)
return map, ''.join(result)
return generate_map(source_name, source_text, find_next_include)
......
......@@ -50,6 +50,12 @@ def run_unittests(command):
if __name__ == "__main__":
lock = threading.Lock()
found = []
if run_cmd(['pypy3', '-V']):
found.append('pypy3 ')
elif run_cmd(['pypy36', '-V']):
found.append('pypy36 ')
elif run_cmd(['pypy', '-V']):
found.append('pypy ')
if run_cmd(['python', '-V']):
output = subprocess.run(['python', '-V'], capture_output=True).stdout
if output.find(b'Python 3') >= 0:
......@@ -58,10 +64,10 @@ if __name__ == "__main__":
found.append('python3')
elif run_cmd(['python3', '-V']):
found.append('python3')
if run_cmd(['python3.5', '-V']):
found.append('python3.5 ')
elif run_cmd(['~/.local/bin/python3.5', '-V']):
found.append('~/.local/bin/python3.5 ')
# if run_cmd(['python3.5', '-V']):
# found.append('python3.5 ')
# elif run_cmd(['~/.local/bin/python3.5', '-V']):
# found.append('~/.local/bin/python3.5 ')
if run_cmd(['python3.6', '-V']):
found.append('python3.6 ')
elif run_cmd(['~/.local/bin/python3.6', '-V']):
......@@ -74,12 +80,6 @@ if __name__ == "__main__":
found.append('python3.8 ')
elif run_cmd(['~/.local/bin/python3.8', '-V']):
found.append('~/.local/bin/python3.8 ')
if run_cmd(['pypy3', '-V']):
found.append('pypy3 ')
elif run_cmd(['pypy36', '-V']):
found.append('pypy36 ')
elif run_cmd(['pypy', '-V']):
found.append('pypy ')
print('Interpreters found: ' + ''.join(found))
arguments = [arg for arg in sys.argv[1:] if arg[:1] != '-']
......
......@@ -26,7 +26,7 @@ import subprocess
import sys
import time
scriptpath = os.path.dirname(__file__) or '.'
scriptpath = os.path.abspath(os.path.dirname(__file__) or '.')
sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
from functools import partial
......@@ -264,11 +264,11 @@ class TestIncludes:
self.create_files({'main.txt': main, 'sub.txt': sub})
find_func = generate_find_include_func(r'include\((?P<name>[^)\n]*)\)')
text, mapping = preprocess_includes('main.txt', None, find_func)
print(mapping)
# print(mapping)
assert text == main.replace('include(sub.txt)', 'abc'), text
for i in range(len(text)):
name, k = mapping(i)
print(i, k, name)
# print(i, k, name)
txt = main if name == 'main.txt' else sub
assert text[i] == txt[k], f'{i}: {text[i]} != {txt[k]} in {name}'
......@@ -280,7 +280,7 @@ class TestIncludes:
perform('012include(sub.txt)xyzinclude(sub.txt)hij', 'abc')
perform('012include(sub.txt)include(sub.txt)hij', 'abc')
perform('include(sub.txt)include(sub.txt)hijinclude(sub.txt)', 'abc')
perform('012include(sub.txt)hilinclude(sub.txt)include(sub.txt)', 'abc')
if __name__ == "__main__":
# tp = TestTokenParsing()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment