Commit 737cd45b authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

preprocess.py: includes: simple test cases completed

parent 05e4860e
...@@ -38,7 +38,7 @@ The central class of module DHParser's ``error`` is the ``Error``-class. ...@@ -38,7 +38,7 @@ The central class of module DHParser's ``error`` is the ``Error``-class.
The easiest way to create an error object is by instantiating The easiest way to create an error object is by instantiating
the Error class with an error message and a source position:: the Error class with an error message and a source position::
>>> error = Error('Somethigs went wrong', 123) >>> error = Error('Something went wrong', 123)
>>> print(error) >>> print(error)
Error (1000): Something went wrong Error (1000): Something went wrong
......
...@@ -337,52 +337,104 @@ def generate_include_map(source_name: str, ...@@ -337,52 +337,104 @@ def generate_include_map(source_name: str,
find_next_include: FindIncludeFunc) -> Tuple[IncludeMap, str]: find_next_include: FindIncludeFunc) -> Tuple[IncludeMap, str]:
file_names: set = set() file_names: set = set()
# def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
# nonlocal file_names
# map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
# text_chunks: List[str] = []
# source_offset: int = 0
# if source_name in file_names:
# raise ValueError(f'Circular include of {source_name} detected!')
# file_names.add(source_name)
# last_begin = -1
# last_end = 0
# lengths = 0
# begin, length, include_name = find_next(source_text, 0)
# while begin >= 0:
# assert begin > last_begin
# with open(include_name, 'r', encoding='utf-8') as f:
# include_text = f.read()
# inner_map, inner_text = generate_map(include_name, include_text, find_next)
# inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
# inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
# if begin == map.positions[-1]: # FEHLER!
# map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
# map.positions = map.positions[:-1] + inner_map.positions[:-1]
# map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
# text_chunks.append(inner_text)
# else:
# text_chunks.append(source_text[last_end:begin])
# source_offset += begin - last_end
# map.file_names += inner_map.file_names[:-1]
# map.positions += inner_map.positions[:-1]
# map.offsets += inner_map.offsets[:-1]
# text_chunks.append(inner_text)
# lengths += length
# map.file_names.append(source_name)
# map.positions.append(inner_map.positions[-1])
# map.offsets.append(source_offset + lengths - inner_map.positions[-1])
# last_end = begin + length
# last_begin = begin
# begin, length, include_name = find_next(source_text, last_end)
# rest = source_text[last_end:]
# if rest:
# text_chunks.append(rest)
# map.positions.append(map.positions[-1] + len(rest))
# map.offsets.append(map.offsets[-1])
# map.file_names.append(source_name)
# file_names.remove(source_name)
# return map, ''.join(text_chunks)
def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]: def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
nonlocal file_names nonlocal file_names
map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name]) map = IncludeMap(source_name, [0], [0], [source_name])
text_chunks: List[str] = [] result = []
source_offset: int = 0
if source_name in file_names: if source_name in file_names:
raise ValueError(f'Circular include of {source_name} detected!') raise ValueError(f'Circular include of {source_name} detected!')
file_names.add(source_name) file_names.add(source_name)
source_pointer = 0
source_offset = 0
result_pointer = 0
last_begin = -1 last_begin = -1
last_end = 0
lengths = 0
begin, length, include_name = find_next(source_text, 0) begin, length, include_name = find_next(source_text, 0)
while begin >= 0: while begin >= 0:
assert begin > last_begin assert begin > last_begin
source_delta = begin - source_pointer
source_pointer += source_delta
result_pointer += source_delta
with open(include_name, 'r', encoding='utf-8') as f: with open(include_name, 'r', encoding='utf-8') as f:
include_text = f.read() included_text = f.read()
inner_map, inner_text = generate_map(include_name, include_text, find_next) inner_map, inner_text = generate_map(include_name, included_text, find_next)
inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions] inner_map.positions = [pos + result_pointer for pos in inner_map.positions]
inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets] inner_map.offsets = [offset - result_pointer for offset in inner_map.offsets]
if begin == map.positions[-1]: # FEHLER! if source_delta == 0:
map.file_names = map.file_names[:-1] + inner_map.file_names[:-1] map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
map.positions = map.positions[:-1] + inner_map.positions[:-1] map.positions = map.positions[:-1] + inner_map.positions[:-1]
map.offsets = map.offsets[:-1] + inner_map.offsets[:-1] map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
text_chunks.append(inner_text) result.append(inner_text)
else: else:
text_chunks.append(source_text[last_end:begin]) result.append(source_text[source_pointer - source_delta: source_pointer])
source_offset += begin - last_end
map.file_names += inner_map.file_names[:-1] map.file_names += inner_map.file_names[:-1]
map.positions += inner_map.positions[:-1] map.positions += inner_map.positions[:-1]
map.offsets += inner_map.offsets[:-1] map.offsets += inner_map.offsets[:-1]
text_chunks.append(inner_text) result.append(inner_text)
lengths += length inner_length = len(inner_text)
result_pointer += inner_length
map.file_names.append(source_name) map.file_names.append(source_name)
map.positions.append(inner_map.positions[-1]) map.positions.append(result_pointer)
map.offsets.append(source_offset + lengths - inner_map.positions[-1]) source_pointer += length
last_end = begin + length source_offset += length - inner_length
last_begin = begin map.offsets.append(source_offset)
begin, length, include_name = find_next(source_text, last_end) begin, length, include_name = find_next(source_text, source_pointer)
rest = source_text[last_end:] rest = source_text[source_pointer:]
if rest: if rest:
text_chunks.append(rest) result.append(rest)
map.positions.append(map.positions[-1] + len(rest)) map.positions.append(map.positions[-1] + len(rest))
map.offsets.append(map.offsets[-1]) map.offsets.append(source_offset)
map.file_names.append(source_name) map.file_names.append(source_name)
file_names.remove(source_name) file_names.remove(source_name)
return map, ''.join(text_chunks) return map, ''.join(result)
return generate_map(source_name, source_text, find_next_include) return generate_map(source_name, source_text, find_next_include)
......
...@@ -50,6 +50,12 @@ def run_unittests(command): ...@@ -50,6 +50,12 @@ def run_unittests(command):
if __name__ == "__main__": if __name__ == "__main__":
lock = threading.Lock() lock = threading.Lock()
found = [] found = []
if run_cmd(['pypy3', '-V']):
found.append('pypy3 ')
elif run_cmd(['pypy36', '-V']):
found.append('pypy36 ')
elif run_cmd(['pypy', '-V']):
found.append('pypy ')
if run_cmd(['python', '-V']): if run_cmd(['python', '-V']):
output = subprocess.run(['python', '-V'], capture_output=True).stdout output = subprocess.run(['python', '-V'], capture_output=True).stdout
if output.find(b'Python 3') >= 0: if output.find(b'Python 3') >= 0:
...@@ -58,10 +64,10 @@ if __name__ == "__main__": ...@@ -58,10 +64,10 @@ if __name__ == "__main__":
found.append('python3') found.append('python3')
elif run_cmd(['python3', '-V']): elif run_cmd(['python3', '-V']):
found.append('python3') found.append('python3')
if run_cmd(['python3.5', '-V']): # if run_cmd(['python3.5', '-V']):
found.append('python3.5 ') # found.append('python3.5 ')
elif run_cmd(['~/.local/bin/python3.5', '-V']): # elif run_cmd(['~/.local/bin/python3.5', '-V']):
found.append('~/.local/bin/python3.5 ') # found.append('~/.local/bin/python3.5 ')
if run_cmd(['python3.6', '-V']): if run_cmd(['python3.6', '-V']):
found.append('python3.6 ') found.append('python3.6 ')
elif run_cmd(['~/.local/bin/python3.6', '-V']): elif run_cmd(['~/.local/bin/python3.6', '-V']):
...@@ -74,12 +80,6 @@ if __name__ == "__main__": ...@@ -74,12 +80,6 @@ if __name__ == "__main__":
found.append('python3.8 ') found.append('python3.8 ')
elif run_cmd(['~/.local/bin/python3.8', '-V']): elif run_cmd(['~/.local/bin/python3.8', '-V']):
found.append('~/.local/bin/python3.8 ') found.append('~/.local/bin/python3.8 ')
if run_cmd(['pypy3', '-V']):
found.append('pypy3 ')
elif run_cmd(['pypy36', '-V']):
found.append('pypy36 ')
elif run_cmd(['pypy', '-V']):
found.append('pypy ')
print('Interpreters found: ' + ''.join(found)) print('Interpreters found: ' + ''.join(found))
arguments = [arg for arg in sys.argv[1:] if arg[:1] != '-'] arguments = [arg for arg in sys.argv[1:] if arg[:1] != '-']
......
...@@ -26,7 +26,7 @@ import subprocess ...@@ -26,7 +26,7 @@ import subprocess
import sys import sys
import time import time
scriptpath = os.path.dirname(__file__) or '.' scriptpath = os.path.abspath(os.path.dirname(__file__) or '.')
sys.path.append(os.path.abspath(os.path.join(scriptpath, '..'))) sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
from functools import partial from functools import partial
...@@ -264,11 +264,11 @@ class TestIncludes: ...@@ -264,11 +264,11 @@ class TestIncludes:
self.create_files({'main.txt': main, 'sub.txt': sub}) self.create_files({'main.txt': main, 'sub.txt': sub})
find_func = generate_find_include_func(r'include\((?P<name>[^)\n]*)\)') find_func = generate_find_include_func(r'include\((?P<name>[^)\n]*)\)')
text, mapping = preprocess_includes('main.txt', None, find_func) text, mapping = preprocess_includes('main.txt', None, find_func)
print(mapping) # print(mapping)
assert text == main.replace('include(sub.txt)', 'abc'), text assert text == main.replace('include(sub.txt)', 'abc'), text
for i in range(len(text)): for i in range(len(text)):
name, k = mapping(i) name, k = mapping(i)
print(i, k, name) # print(i, k, name)
txt = main if name == 'main.txt' else sub txt = main if name == 'main.txt' else sub
assert text[i] == txt[k], f'{i}: {text[i]} != {txt[k]} in {name}' assert text[i] == txt[k], f'{i}: {text[i]} != {txt[k]} in {name}'
...@@ -280,7 +280,7 @@ class TestIncludes: ...@@ -280,7 +280,7 @@ class TestIncludes:
perform('012include(sub.txt)xyzinclude(sub.txt)hij', 'abc') perform('012include(sub.txt)xyzinclude(sub.txt)hij', 'abc')
perform('012include(sub.txt)include(sub.txt)hij', 'abc') perform('012include(sub.txt)include(sub.txt)hij', 'abc')
perform('include(sub.txt)include(sub.txt)hijinclude(sub.txt)', 'abc') perform('include(sub.txt)include(sub.txt)hijinclude(sub.txt)', 'abc')
perform('012include(sub.txt)hilinclude(sub.txt)include(sub.txt)', 'abc')
if __name__ == "__main__": if __name__ == "__main__":
# tp = TestTokenParsing() # tp = TestTokenParsing()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment