cleanups finished

DHParser/dsl.py

@@ -169,14 +169,15 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
         # read grammar
         grammar_src = load_if_file(grammar_representation)
         if is_python_code(grammar_src):
-            parser_py, messages, AST = grammar_src, [], None  # type: str, List[Error], Node
+            parser_py, messages = grammar_src, []  # type: str, List[Error]
         else:
             with logging(False):
-                parser_py, messages, AST = compile_source(grammar_src, None,
+                parser_py, messages, _ = compile_source(
+                    grammar_src, None,
                     get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
         if has_errors(messages):
             raise GrammarError(only_errors(messages), grammar_src)
-        parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w+Grammar$')()
+        parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, r'\w+Grammar$')()
     else:
         # assume that dsl_grammar is a ParserHQ-object or Grammar class
         grammar_src = ''
@@ -197,7 +198,7 @@ def compileDSL(text_or_file: str,
     Compiles a text in a domain specific language (DSL) with an
     EBNF-specified grammar. Returns the compiled text or raises a
     compilation error.
-    
+
     Raises:
         CompilationError if any errors occurred during compilation
     """
@@ -218,16 +219,16 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
     Compiles an EBNF grammar file and returns the compiler object
     that was used and which can now be queried for the result as well
     as skeleton code for preprocessor, transformer and compiler objects.
-    
+
     Args:
         ebnf_src(str):  Either the file name of an EBNF grammar or
             the EBNF grammar itself as a string.
         branding (str):  Branding name for the compiler suite source
-            code. 
+            code.
     Returns:
         An instance of class ``ebnf.EBNFCompiler``
     Raises:
-        CompilationError if any errors occurred during compilation      
+        CompilationError if any errors occurred during compilation
     """
     grammar = get_ebnf_grammar()
     compiler = get_ebnf_compiler(branding, ebnf_src)
@@ -246,11 +247,11 @@ def compileEBNF(ebnf_src: str, branding="DSL") -> str:
         ebnf_src(str):  Either the file name of an EBNF grammar or
             the EBNF grammar itself as a string.
         branding (str):  Branding name for the compiler suite source
-            code. 
+            code.
     Returns:
         The complete compiler suite skeleton as Python source code.
     Raises:
-        CompilationError if any errors occurred during compilation        
+        CompilationError if any errors occurred during compilation
     """
     compiler = raw_compileEBNF(ebnf_src, branding)
     src = ["#/usr/bin/python\n",
@@ -272,23 +273,23 @@ def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
         ebnf_src(str):  Either the file name of an EBNF grammar or
             the EBNF grammar itself as a string.
         branding (str or bool):  Branding name for the compiler
-            suite source code. 
-    
+            suite source code.
+
     Returns:
         A provider function for a grammar object for texts in the
         language defined by ``ebnf_src``.
     """
     grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
                              get_ebnf_transformer(), get_ebnf_compiler(branding))
-    return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_(?:\w+_)?grammar$')
+    return compile_python_object(DHPARSER_IMPORTS + grammar_src, r'get_(?:\w+_)?grammar$')
 
 
 def load_compiler_suite(compiler_suite: str) -> \
-        Tuple[PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc]:
+    Tuple[PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc]:
     """
     Extracts a compiler suite from file or string ``compiler suite``
     and returns it as a tuple (preprocessor, parser, ast, compiler).
-    
+
     Returns:
         4-tuple (preprocessor function, parser class, ast transformer function, compiler class)
     """
@@ -298,26 +299,27 @@ def load_compiler_suite(compiler_suite: str) -> \
     imports = DHPARSER_IMPORTS
     if is_python_code(compiler_suite):
         try:
-            intro, imports, preprocessor_py, parser_py, ast_py, compiler_py, outro = \
+            _, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = \
                 RX_SECTION_MARKER.split(source)
-        except ValueError as error:
+        except ValueError:
             raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
                                  'Please delete or repair file manually.')
         # TODO: Compile in one step and pick parts from namespace later ?
-        preprocessor = compile_python_object(imports + preprocessor_py, 'get_(?:\w+_)?preprocessor$')
-        parser = compile_python_object(imports + parser_py, 'get_(?:\w+_)?grammar$')
-        ast = compile_python_object(imports + ast_py, 'get_(?:\w+_)?transformer$')
+        preprocessor = compile_python_object(imports + preprocessor_py,
+                                             r'get_(?:\w+_)?preprocessor$')
+        parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
+        ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
     else:
         # assume source is an ebnf grammar. Is there really any reasonable application case for this?
         with logging(False):
-            compile_py, messages, AST = compile_source(source, None, get_ebnf_grammar(),
-                                                       get_ebnf_transformer(), get_ebnf_compiler())
+            compiler_py, messages, _ = compile_source(source, None, get_ebnf_grammar(),
+                                                      get_ebnf_transformer(), get_ebnf_compiler())
         if has_errors(messages):
             raise GrammarError(only_errors(messages), source)
         preprocessor = get_ebnf_preprocessor
         parser = get_ebnf_grammar
         ast = get_ebnf_transformer
-    compiler = compile_python_object(imports + compiler_py, 'get_(?:\w+_)?compiler$')
+    compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
 
     return preprocessor, parser, ast, compiler
 
@@ -325,12 +327,12 @@ def load_compiler_suite(compiler_suite: str) -> \
 def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
     """
     Returns ``True``  if the ``compile_suite`` needs to be updated.
-     
-    An update is needed, if either the grammar in the compieler suite 
-    does not reflect the latest changes of ``grammar_source`` or if 
+
+    An update is needed, if either the grammar in the compieler suite
+    does not reflect the latest changes of ``grammar_source`` or if
     sections from the compiler suite have diligently been overwritten
     with whitespace order to trigger their recreation. Note: Do not
-    delete or overwrite the section marker itself. 
+    delete or overwrite the section marker itself.
 
     Args:
         compiler_suite:  the parser class representing the grammar
@@ -342,7 +344,7 @@ def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
         True, if ``compiler_suite`` seems to be out of date.
     """
     try:
-        preprocessor, grammar, ast, compiler = load_compiler_suite(compiler_suite)
+        _, grammar, _, _ = load_compiler_suite(compiler_suite)
         return grammar_changed(grammar(), grammar_source)
     except ValueError:
         return True
@@ -353,17 +355,17 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
 
     Args:
         text_or_file (str):  Either the file name of the source code or
-            the source code directly. (Which is determined by 
+            the source code directly. (Which is determined by
             heuristics. If ``text_or_file`` contains at least on
             linefeed then it is always assumed to be a source text and
             not a file name.)
         compiler_suite(str):  File name of the compiler suite to be
             used.
-        
+
     Returns:
-        The result of the compilation, the form and type of which 
+        The result of the compilation, the form and type of which
         depends entirely on the compiler.
-        
+
     Raises:
         CompilerError
     """
@@ -398,7 +400,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
             is written to a file with the same name but a different
             extension than the source file. This parameter sets the
             extension.
-            
+
     Returns:
         A (potentially empty) list of error or warning messages.
     """

DHParser/ebnf.py

@@ -430,13 +430,13 @@ class EBNFCompiler(Compiler):
                       '    # AST Transformations for the ' + self.grammar_name + '-grammar']
         transtable.append('    "+": remove_empty,')
         for name in self.rules:
-            tf = '[]'
+            transformations = '[]'
             rule = self.definitions[name]
             if rule.startswith('Alternative'):
-                tf = '[replace_or_reduce]'
+                transformations = '[replace_or_reduce]'
             elif rule.startswith('Synonym'):
-                tf = '[replace_by_single_child]'
-            transtable.append('    "' + name + '": %s,' % tf)
+                transformations = '[replace_by_single_child]'
+            transtable.append('    "' + name + '": %s,' % transformations)
         transtable.append('    ":Token, :RE": reduce_single_child,')
         transtable += ['    "*": replace_by_single_child', '}', '']
         transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
@@ -459,7 +459,7 @@ class EBNFCompiler(Compiler):
                     self.grammar_name + '", grammar_source=""):',
                     '        super(' + self.grammar_name +
                     'Compiler, self).__init__(grammar_name, grammar_source)',
-                    "        assert re.match('\w+\Z', grammar_name)", '']
+                    r"        assert re.match('\w+\Z', grammar_name)", '']
         for name in self.rules:
             method_name = Compiler.method_name(name)
             if name == self.root_symbol:
@@ -548,6 +548,8 @@ class EBNFCompiler(Compiler):
         defined_symbols.difference_update(self.RESERVED_SYMBOLS)
 
         def remove_connections(symbol):
+            """Recursiviely removes all symbols which appear in the
+            definiens of a particular symbol."""
             if symbol in defined_symbols:
                 defined_symbols.remove(symbol)
                 for related in self.rules[symbol][1:]:
@@ -555,8 +557,9 @@ class EBNFCompiler(Compiler):
 
         remove_connections(self.root_symbol)
         for leftover in defined_symbols:
-            self.rules[leftover][0].add_error(('Rule "%s" is not connected to '
-                'parser root "%s" !') % (leftover, self.root_symbol), Error.WARNING)
+            self.rules[leftover][0].add_error(
+                ('Rule "%s" is not connected to parser root "%s" !') % 
+                (leftover, self.root_symbol), Error.WARNING)
 
         # set root_symbol parser and assemble python grammar definition
 
@@ -573,6 +576,12 @@ class EBNFCompiler(Compiler):
     def on_syntax(self, node: Node) -> str:
         definitions = []  # type: List[Tuple[str, str]]
 
+        # transfer directives to re_flags where needed
+        if self.directives['ignorecase']:
+            self.re_flags.add('i')
+        elif 'i' in self.re_flags:
+            self.re_flags.remove('i')
+
         # drop the wrapping sequence node
         if len(node.children) == 1 and not node.children[0].parser.name:
             node = node.children[0]
@@ -678,6 +687,8 @@ class EBNFCompiler(Compiler):
             self.directives['ignorecase'] == value
             if value:
                 self.re_flags.add('i')
+            elif 'i' in self.re_flags:
+                self.re_flags.remove('i')
 
         # elif key == 'testing':
         #     value = str(node.children[1])
@@ -685,14 +696,14 @@ class EBNFCompiler(Compiler):
 
         elif key == 'literalws':
             value = {item.lower() for item in self.compile(node.children[1])}
-            if (len(value - {'left', 'right', 'both', 'none'}) > 0
+            if ((value - {'left', 'right', 'both', 'none'})
                     or ('none' in value and len(value) > 1)):
                 node.add_error('Directive "literalws" allows the values '
                                '`left`, `right`, `both` or `none`, '
                                'but not `%s`' % ", ".join(value))
-            ws = {'left', 'right'} if 'both' in value \
+            wsp = {'left', 'right'} if 'both' in value \
                 else {} if 'none' in value else value
-            self.directives[key] = list(ws)
+            self.directives[key] = list(wsp)
 
         elif key in {'tokens', 'preprocessor_tokens'}:
             self.directives['tokens'] |= self.compile(node.children[1])
@@ -747,7 +758,7 @@ class EBNFCompiler(Compiler):
                 i += 1
         saved_result = node.result
         node.result = tuple(filtered_children)
-        custom_args =  ['mandatory=%i' % mandatory_marker[0]] if mandatory_marker else []
+        custom_args = ['mandatory=%i' % mandatory_marker[0]] if mandatory_marker else []
         compiled = self.non_terminal(node, 'Series', custom_args)
         node.result = saved_result
         return compiled
@@ -832,10 +843,10 @@ class EBNFCompiler(Compiler):
         # return self.non_terminal(node, 'Unordered')
         assert len(node.children) == 1
         nd = node.children[0]
-        for r in nd.children:
-            if r.parser.ptype == TOKEN_PTYPE and str(nd) == "§":
+        for child in nd.children:
+            if child.parser.ptype == TOKEN_PTYPE and str(nd) == "§":
                 node.add_error("Unordered parser lists cannot contain mandatory (§) items.")
-        args = ', '.join(self.compile(r) for r in nd.children)
+        args = ', '.join(self.compile(child) for child in nd.children)
         if nd.parser.name == "term":
             return "AllOf(" + args + ")"
         elif nd.parser.name == "expression":

DHParser/testing.py

@@ -189,8 +189,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 # write parsing-history log only in case of failure!
                 parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
             elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
-                    errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
-                                  (test_name, parser_name, cst.as_sxpr()))
+                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
+                              (test_name, parser_name, cst.as_sxpr()))
             elif "ast" in tests:
                 compare = mock_syntax_tree(tests["ast"][test_name])
                 if compare != ast:
@@ -245,7 +245,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
         print("\nScanning test-directory: " + directory)
     save_cwd = os.getcwd()
     os.chdir(directory)
-    if is_logging():    clear_logs()
+    if is_logging():
+        clear_logs()
     for filename in sorted(os.listdir()):
         if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
             try:
@@ -259,7 +260,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
                 if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
                     raise e
     os.chdir(save_cwd)
-    error_report = [];  err_N = 0
+    error_report = []
+    err_N = 0
     if all_errors:
         for filename in all_errors:
             error_report.append('Errors found by unit test "%s":' % filename)
@@ -267,16 +269,18 @@ def grammar_suite(directory, parser_factory, transformer_factory,
             for error in all_errors[filename]:
                 error_report.append('\t' + '\n\t'.join(error.split('\n')))
     if error_report:
-        if verbose:  print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
+        if verbose:
+            print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
         return ('Test suite "%s" revealed some errors:\n\n' % directory) + '\n'.join(error_report)
-    if verbose:  print("\nSUCCESS! All tests passed :-)\n")
+    if verbose:
+        print("\nSUCCESS! All tests passed :-)\n")
     return ''
 
 
 def runner(tests, namespace):
     """
     Runs all or some selected Python unit tests found in the
-    namespace. To run all tests in a module, call 
+    namespace. To run all tests in a module, call
     ``runner("", globals())`` from within that module.
 
     Args:
@@ -284,9 +288,9 @@ def runner(tests, namespace):
             names of test or test classes. Each test and, in the case
             of a test class, all tests within the test class will be
             run.
-        namespace: The namespace for running the test, usually 
+        namespace: The namespace for running the test, usually
             ``globals()`` should be used.
-            
+
     Example:
         class TestSomething()
             def setup(self):
@@ -295,10 +299,10 @@ def runner(tests, namespace):
                 pass
             def test_something(self):
                 pass
-                
+
         if __name__ == "__main__":
             from DHParser.testing import runner
-            runner("", globals())        
+            runner("", globals())
     """
     def instantiate(cls_name):
         exec("obj = " + cls_name + "()", namespace)