- source_map test

DHParser/preprocess.py

@@ -19,10 +19,10 @@ permissions and limitations under the License.
 import bisect
 import collections
 import functools
-
-from DHParser.toolkit import typing, re
 from typing import Union, Callable
 
+from DHParser.toolkit import re
+
 __all__ = ('RX_TOKEN_NAME',
            'BEGIN_TOKEN',
            'TOKEN_DELIMITER',
@@ -102,9 +102,9 @@ def tokenized_to_original_mapping(tokenized_source: str) -> SourceMap:
         d = tokenized_source.find(TOKEN_DELIMITER, i)
         e = tokenized_source.find(END_TOKEN, i)
         assert 0 <= d < e
-        o -= (d - i + 2)
+        o -= (d - i + 3)
         positions.extend([d + 1, e + 1])
-        offsets.extend([o, o - 1])
+        offsets.extend([o + 1, o])
         i = tokenized_source.find(BEGIN_TOKEN, e + 1)
 
     # post conditions

test/test_preprocess.py

@@ -22,10 +22,10 @@ limitations under the License.
 # import sys
 # sys.path.append('../')
 
-from DHParser.toolkit import re, lstrip_docstring, logging
-from DHParser.preprocess import make_token, tokenized_to_original_mapping, source_map, \
-    BEGIN_TOKEN, END_TOKEN, TOKEN_DELIMITER, pp_tokenized
 from DHParser.dsl import grammar_provider
+from DHParser.preprocess import make_token, tokenized_to_original_mapping, source_map, \
+    BEGIN_TOKEN, END_TOKEN, TOKEN_DELIMITER
+from DHParser.toolkit import lstrip_docstring
 
 
 class TestMakeToken:
@@ -109,6 +109,14 @@ class TestTokenParsing:
         self.tokenized = self.preprocess_indentation(self.code)
         self.srcmap = tokenized_to_original_mapping(self.tokenized)
 
+    def verify_mapping(self, teststr, orig_text, preprocessed_text):
+        mapped_pos = preprocessed_text.find(teststr)
+        assert mapped_pos >= 0
+        original_pos = source_map(mapped_pos, self.srcmap)
+        assert orig_text[original_pos:original_pos + len(teststr)] == teststr, \
+            '"%s" (%i) wrongly mapped onto "%s" (%i)' % \
+            (teststr, mapped_pos, orig_text[original_pos:original_pos + len(teststr)], original_pos)
+
     def test_parse_tokenized(self):
         cst = self.grammar(self.tokenized)
         # for e in cst.collect_errors(self.tokenized):
@@ -116,6 +124,13 @@ class TestTokenParsing:
         #     print()
         assert not cst.error_flag
 
+    def test_source_mapping(self):
+        self.verify_mapping("def func", self.code, self.tokenized)
+        self.verify_mapping("x > 0:", self.code, self.tokenized)
+        self.verify_mapping("if y > 0:", self.code, self.tokenized)
+        self.verify_mapping("print(x)", self.code, self.tokenized)
+        self.verify_mapping("print(y)", self.code, self.tokenized)
+
 
 if __name__ == "__main__":
     # tp = TestTokenParsing()