preprocess.py: includes: support for comments added

DHParser/preprocess.py

@@ -309,27 +309,43 @@ def with_source_mapping(result: PreprocessorResult) -> Preprocessed:
 
 #######################################################################
 #
-# Includes - support for chaining source texts
-#
-# NOT YET TESTED!!!
+# Includes - support for chaining source texts via an in clude command
 #
 #######################################################################
 
 
-def generate_find_include_func(rx: Union[str, Any]) -> FindIncludeFunc:
+def generate_find_include_func(rx: Union[str, Any],
+                               comment_rx: Optional[Union[str, Any]] = None) -> FindIncludeFunc:
     if isinstance(rx, str):  rx = re.compile(rx)
+    if isinstance(comment_rx, str):  comment_rx = re.compile(comment_rx)
 
     def find_include(text: str, begin: int) -> IncludeInfo:
         nonlocal rx
-        iterator = rx.finditer(text, begin)
-        try:
-            m = next(iterator)
+        m = rx.search(text, begin)
+        if m:
             begin = m.start()
             return IncludeInfo(begin, m.end() - begin, m.group('name'))
-        except StopIteration:
+        else:
             return IncludeInfo(-1, 0, '')
 
-    return find_include
+    def find_comment(text: str, begin: int) -> Tuple[int, int]:
+        m = comment_rx.search(text, begin)
+        return m.span() if m else (-1, -2)
+
+    def meta_find_include(text: str, begin: int) -> IncludeInfo:
+        a, b = find_comment(text, begin)
+        info = find_include(text, begin)
+        k, length, name = info
+        while a < b <= k:
+            a, b = find_comment(text, b)
+        while (a < k < b) or (a < k + length < b):
+            info = find_include(text, b)
+            k, length, name = info
+            while a < b <= k:
+                a, b = find_comment(text, b)
+        return info
+
+    return find_include if comment_rx is None else meta_find_include
 
 
 def generate_include_map(source_name: str,
@@ -337,53 +353,6 @@ def generate_include_map(source_name: str,
                          find_next_include: FindIncludeFunc) -> Tuple[IncludeMap, str]:
     file_names: set = set()
 
-    # def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
-    #     nonlocal file_names
-    #     map: IncludeMap = IncludeMap(source_name, [0], [0], [source_name])
-    #     text_chunks: List[str] = []
-    #     source_offset: int = 0
-    #     if source_name in file_names:
-    #         raise ValueError(f'Circular include of {source_name} detected!')
-    #     file_names.add(source_name)
-    #     last_begin = -1
-    #     last_end = 0
-    #     lengths = 0
-    #     begin, length, include_name = find_next(source_text, 0)
-    #     while begin >= 0:
-    #         assert begin > last_begin
-    #         with open(include_name, 'r', encoding='utf-8') as f:
-    #             include_text = f.read()
-    #         inner_map, inner_text = generate_map(include_name, include_text, find_next)
-    #         inner_map.positions = [pos + begin - lengths + source_offset for pos in inner_map.positions]
-    #         inner_map.offsets = [offset - (source_offset + begin - lengths) for offset in inner_map.offsets]
-    #         if begin == map.positions[-1]:  # FEHLER!
-    #             map.file_names = map.file_names[:-1] + inner_map.file_names[:-1]
-    #             map.positions = map.positions[:-1] + inner_map.positions[:-1]
-    #             map.offsets = map.offsets[:-1] + inner_map.offsets[:-1]
-    #             text_chunks.append(inner_text)
-    #         else:
-    #             text_chunks.append(source_text[last_end:begin])
-    #             source_offset += begin - last_end
-    #             map.file_names += inner_map.file_names[:-1]
-    #             map.positions += inner_map.positions[:-1]
-    #             map.offsets += inner_map.offsets[:-1]
-    #             text_chunks.append(inner_text)
-    #         lengths += length
-    #         map.file_names.append(source_name)
-    #         map.positions.append(inner_map.positions[-1])
-    #         map.offsets.append(source_offset + lengths - inner_map.positions[-1])
-    #         last_end = begin + length
-    #         last_begin = begin
-    #         begin, length, include_name = find_next(source_text, last_end)
-    #     rest = source_text[last_end:]
-    #     if rest:
-    #         text_chunks.append(rest)
-    #         map.positions.append(map.positions[-1] + len(rest))
-    #         map.offsets.append(map.offsets[-1])
-    #         map.file_names.append(source_name)
-    #     file_names.remove(source_name)
-    #     return map, ''.join(text_chunks)
-
     def generate_map(source_name, source_text, find_next) -> Tuple[IncludeMap, str]:
         nonlocal file_names
         map = IncludeMap(source_name, [0], [0], [source_name])

tests/test_preprocess.py

@@ -223,6 +223,19 @@ class TestHelpers:
         info = find('''321include(sub.txt)xyz''', 0)
         assert info == IncludeInfo(3, 16, 'sub.txt')
 
+    def test_generate_find_include_w_comments(self):
+        rx = re.compile(r'include\((?P<name>[^)\n]*)\)')
+        comment_rx = re.compile(r'#.*(?:\n|$)')
+        find = generate_find_include_func(rx, comment_rx)
+        test = '''a
+        b # include(alpha)
+        c include(beta)
+        # include(gamma)'''
+        info = find(test, 0)
+        assert info.file_name == "beta"
+        info = find(test, info.begin + info.length)
+        assert info.begin < 0
+
 
 def system(s: str) -> int:
     # return os.system(s)