preprocess.py: aglorithm for includes seems to work now...

DHParser/error.py

@@ -408,31 +408,11 @@ def adjust_error_locations(errors: List[Error],
     for err in errors:
         assert err.pos >= 0
         err.orig_doc, lbreaks, err.orig_pos = source_mapping(err.pos)
-        err.line, err.column = line_col(line_breaks, err.orig_pos)
-        # adjust length in case it exceeds the text size. As this is non-fatal
-        # it should be adjusted rather than an error raised to avoid
-        # unnecessary special-case treatments in other places
+        err.line, err.column = line_col(lbreaks, err.orig_pos)
         if err.orig_pos + err.length > len(err.orig_doc):
-            err.length = len(err.orig_doc) - err.orig_pos
+            err.length = len(err.orig_doc) - err.orig_pos  # err.length should not exceed text length
         err.end_line, err.end_column = line_col(lbreaks, err.orig_pos + err.length)
 
-# def canonical_error_strings(errors: List[Error], source_file_name: str = '') -> List[str]:
-#     """Returns the list of error strings in canonical form that can be parsed by most
-#     editors, i.e. "relative filepath : line : column : severity (code) : error string"
-#     """
-#     if errors:
-#         if is_filename(source_file_name):
-#             cwd = os.getcwd()
-#             if source_file_name.startswith(cwd):
-#                 rel_path = source_file_name[len(cwd)]
-#             else:
-#                 rel_path = source_file_name
-#             error_strings = [rel_path + ':' + str(err) for err in errors]
-#         else:
-#             error_strings = [str(err) for err in errors]
-#     else:
-#         error_strings = []
-#     return error_strings
 
 def canonical_error_strings(errors: List[Error]) -> List[str]:
     """Returns the list of error strings in canonical form that can be parsed by most

DHParser/parse.py

@@ -45,7 +45,7 @@ from DHParser.error import Error, ErrorCode, MANDATORY_CONTINUATION, \
     OPTIONAL_REDUNDANTLY_NESTED_WARNING, CAPTURE_STACK_NOT_EMPTY, BAD_REPETITION_COUNT, \
     AUTORETRIEVED_SYMBOL_NOT_CLEARED, RECURSION_DEPTH_LIMIT_HIT
 from DHParser.log import CallItem, HistoryRecord
-from DHParser.preprocess import BEGIN_TOKEN, END_TOKEN, RX_TOKEN_NAME
+from DHParser.preprocess import BEGIN_TOKEN, END_TOKEN, RX_TOKEN_NAME, SourceMapFunc
 from DHParser.stringview import StringView, EMPTY_STRING_VIEW
 from DHParser.syntaxtree import ChildrenType, Node, RootNode, WHITESPACE_PTYPE, \
     TOKEN_PTYPE, ZOMBIE_TAG, EMPTY_NODE, ResultType
@@ -1361,7 +1361,7 @@ class Grammar:
     def __call__(self,
                  document: str,
                  start_parser: Union[str, Parser] = "root_parser__",
-                 source_mapping = identity,
+                 source_mapping: Optional[SourceMapFunc] = None,
                  *, complete_match: bool = True) -> RootNode:
         """
         Parses a document with with parser-combinators.

DHParser/syntaxtree.py

@@ -2758,7 +2758,7 @@ class RootNode(Node):
 
     def swallow(self, node: Optional[Node],
                 source: Union[str, StringView] = '',
-                source_mapping: SourceMapFunc = identity) \
+                source_mapping: Optional[SourceMapFunc] = None) \
             -> 'RootNode':
         """
         Put `self` in the place of `node` by copying all its data.
@@ -2774,7 +2774,11 @@ class RootNode(Node):
         if source and source != self.source:
             self.source = source
             self.lbreaks = linebreaks(source)
-        if source_mapping != identity:  self.source_mapping = source_mapping
+        if source_mapping is None:
+            line_breaks = linebreaks(source)
+            self.source_mapping = lambda pos: SourceLocation('', line_breaks, pos)
+        else:
+            self.source_mapping = source_mapping  # type: SourceMapFunc
         if self.tag_name != '__not_yet_ready__':
             raise AssertionError('RootNode.swallow() has already been called!')
         if node is None:

tests/test_preprocess.py

@@ -301,20 +301,15 @@ class TestIncludes:
             self.create_files(ensemble)
             find_func = gen_find_include_func(r'#include\((?P<name>[^)\n]*)\)')
             text, mapping = preprocess_includes(None, 'main', find_func)
-            # print(mapping)
             substrings = {}
             for k, v in reversed(list(ensemble.items())):
                 for name, content in substrings.items():
                     v = v.replace(f'#include({name})', content)
                 substrings[k] = v
             assert text == substrings['main']
-            # print(text)
             for i in range(len(text)):
                 name, lbreaks, k = mapping(i)
                 txt = ensemble[name]
-                # print(name, substrings[name], text[offset:offset + len(substrings[name])])
-                # assert text[offset:offset + len(substrings[name])] == substrings[name]
-                # print(name, txt, i, k)
                 assert text[i] == txt[k], f'{i}: {text[i]} != {txt[k]} in {name}'
 
         perform(main = '#include(sub)xyz', sub='abc')