preprocess.py: includes: simple test cases completed

DHParser/error.py

@@ -38,7 +38,7 @@ The central class of module DHParser's ``error``  is the ``Error``-class.
 The easiest way to create an error object is by instantiating
 the Error class with an error message and a source position::
 
-    >>> error = Error('Somethigs went wrong', 123)
+    >>> error = Error('Something went wrong', 123)
     >>> print(error)
     Error (1000): Something went wrong
 

DHParser/preprocess.py

@@ -337,52 +337,104 @@ def generate_include_map(source_name: str,
                          find_next_include: FindIncludeFunc) -> Tuple[IncludeMap, str]:
     file_names: set = set()
 
+    # def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
+    #     nonlocal file_names
+    #     map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
+    #     text_chunks: List[str] = []
+    #     source_offset: int = 0
+    #     if source_name in file_names:
+    #         raise ValueError(f'Circular include of {source_name} detected!')
+    #     file_names.add(source_name)
+    #     last_begin = -1
+    #     last_end = 0
+    #     lengths = 0
+    #     begin, length, include_name = find_next(source_text, 0)
+    #     while begin >= 0:
+    #         assert begin > last_begin
+    #         with open(include_name, 'r', encoding='utf-8') as f:
+    #             include_text = f.read()
+    #         inner_map, inner_text = generate_map(include_name, include_text, find_next)
+    #         inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
+    #         inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
+    #         if begin == map.positions[-1]:  # FEHLER!
+    #             map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
+    #             map.positions = map.positions[:-1] + inner_map.positions[:-1]
+    #             map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
+    #             text_chunks.append(inner_text)
+    #         else:
+    #             text_chunks.append(source_text[last_end:begin])
+    #             source_offset += begin - last_end
+    #             map.file_names += inner_map.file_names[:-1]
+    #             map.positions += inner_map.positions[:-1]
+    #             map.offsets += inner_map.offsets[:-1]
+    #             text_chunks.append(inner_text)
+    #         lengths += length
+    #         map.file_names.append(source_name)
+    #         map.positions.append(inner_map.positions[-1])
+    #         map.offsets.append(source_offset + lengths - inner_map.positions[-1])
+    #         last_end = begin + length
+    #         last_begin = begin
+    #         begin, length, include_name = find_next(source_text, last_end)
+    #     rest = source_text[last_end:]
+    #     if rest:
+    #         text_chunks.append(rest)
+    #         map.positions.append(map.positions[-1] + len(rest))
+    #         map.offsets.append(map.offsets[-1])
+    #         map.file_names.append(source_name)
+    #     file_names.remove(source_name)
+    #     return map, ''.join(text_chunks)
+
     def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
         nonlocal file_names
-        map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
-        text_chunks: List[str] = []
-        source_offset: int = 0
+        map = IncludeMap(source_name, [0], [0], [source_name])
+        result = []
+
         if source_name in file_names:
             raise ValueError(f'Circular include of {source_name} detected!')
         file_names.add(source_name)
+
+        source_pointer = 0
+        source_offset = 0
+        result_pointer = 0
         last_begin = -1
-        last_end = 0
-        lengths = 0
         begin, length, include_name = find_next(source_text, 0)
         while begin >= 0:
             assert begin > last_begin
+            source_delta = begin - source_pointer
+            source_pointer += source_delta
+            result_pointer += source_delta
             with open(include_name, 'r', encoding='utf-8') as f:
-                include_text = f.read()
-            inner_map, inner_text = generate_map(include_name, include_text, find_next)
-            inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
-            inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
-            if begin == map.positions[-1]:  # FEHLER!
+                included_text = f.read()
+            inner_map, inner_text = generate_map(include_name, included_text, find_next)
+            inner_map.positions = [pos + result_pointer for pos in inner_map.positions]
+            inner_map.offsets = [offset - result_pointer for offset in inner_map.offsets]
+            if source_delta == 0:
                 map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
                 map.positions = map.positions[:-1] + inner_map.positions[:-1]
                 map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
-                text_chunks.append(inner_text)
+                result.append(inner_text)
             else:
-                text_chunks.append(source_text[last_end:begin])
-                source_offset += begin - last_end
+                result.append(source_text[source_pointer - source_delta: source_pointer])
                 map.file_names += inner_map.file_names[:-1]
                 map.positions += inner_map.positions[:-1]
                 map.offsets += inner_map.offsets[:-1]
-                text_chunks.append(inner_text)
-            lengths += length
+                result.append(inner_text)
+            inner_length = len(inner_text)
+            result_pointer += inner_length
             map.file_names.append(source_name)
-            map.positions.append(inner_map.positions[-1])
-            map.offsets.append(source_offset + lengths - inner_map.positions[-1])
-            last_end = begin + length
-            last_begin = begin
-            begin, length, include_name = find_next(source_text, last_end)
-        rest = source_text[last_end:]
+            map.positions.append(result_pointer)
+            source_pointer += length
+            source_offset += length - inner_length
+            map.offsets.append(source_offset)
+            begin, length, include_name = find_next(source_text, source_pointer)
+        rest = source_text[source_pointer:]
         if rest:
-            text_chunks.append(rest)
+            result.append(rest)
             map.positions.append(map.positions[-1] + len(rest))
-            map.offsets.append(map.offsets[-1])
+            map.offsets.append(source_offset)
             map.file_names.append(source_name)
         file_names.remove(source_name)
-        return map, ''.join(text_chunks)
+        return map, ''.join(result)
 
     return generate_map(source_name, source_text, find_next_include)
 

tests/run.py

@@ -50,6 +50,12 @@ def run_unittests(command):
 if __name__ == "__main__":
     lock = threading.Lock()
     found = []
+    if run_cmd(['pypy3', '-V']):
+        found.append('pypy3 ')
+    elif run_cmd(['pypy36', '-V']):
+        found.append('pypy36 ')
+    elif run_cmd(['pypy', '-V']):
+        found.append('pypy ')
     if run_cmd(['python', '-V']):
         output = subprocess.run(['python', '-V'], capture_output=True).stdout
         if output.find(b'Python 3') >= 0:
@@ -58,10 +64,10 @@ if __name__ == "__main__":
             found.append('python3')
     elif run_cmd(['python3', '-V']):
         found.append('python3')
-    if run_cmd(['python3.5', '-V']):
-        found.append('python3.5 ')
-    elif run_cmd(['~/.local/bin/python3.5', '-V']):
-        found.append('~/.local/bin/python3.5 ')
+    # if run_cmd(['python3.5', '-V']):
+    #     found.append('python3.5 ')
+    # elif run_cmd(['~/.local/bin/python3.5', '-V']):
+    #     found.append('~/.local/bin/python3.5 ')
     if run_cmd(['python3.6', '-V']):
         found.append('python3.6 ')
     elif run_cmd(['~/.local/bin/python3.6', '-V']):
@@ -74,12 +80,6 @@ if __name__ == "__main__":
         found.append('python3.8 ')
     elif run_cmd(['~/.local/bin/python3.8', '-V']):
         found.append('~/.local/bin/python3.8 ')
-    if run_cmd(['pypy3', '-V']):
-        found.append('pypy3 ')
-    elif run_cmd(['pypy36', '-V']):
-        found.append('pypy36 ')
-    elif run_cmd(['pypy', '-V']):
-        found.append('pypy ')
     print('Interpreters found: ' + ''.join(found))
 
     arguments = [arg for arg in sys.argv[1:] if arg[:1] != '-']

tests/test_preprocess.py

@@ -26,7 +26,7 @@ import subprocess
 import sys
 import time
 
-scriptpath = os.path.dirname(__file__) or '.'
+scriptpath = os.path.abspath(os.path.dirname(__file__) or '.')
 sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
 
 from functools import partial
@@ -264,11 +264,11 @@ class TestIncludes:
             self.create_files({'main.txt': main, 'sub.txt': sub})
             find_func = generate_find_include_func(r'include\((?P<name>[^)\n]*)\)')
             text, mapping = preprocess_includes('main.txt', None, find_func)
-            print(mapping)
+            # print(mapping)
             assert text == main.replace('include(sub.txt)', 'abc'), text
             for i in range(len(text)):
                 name, k = mapping(i)
-                print(i, k, name)
+                # print(i, k, name)
                 txt = main if name == 'main.txt' else sub
                 assert text[i] == txt[k], f'{i}: {text[i]} != {txt[k]} in {name}'
 
@@ -280,7 +280,7 @@ class TestIncludes:
         perform('012include(sub.txt)xyzinclude(sub.txt)hij', 'abc')
         perform('012include(sub.txt)include(sub.txt)hij', 'abc')
         perform('include(sub.txt)include(sub.txt)hijinclude(sub.txt)', 'abc')
-
+        perform('012include(sub.txt)hilinclude(sub.txt)include(sub.txt)', 'abc')
 
 if __name__ == "__main__":
     # tp = TestTokenParsing()