- scripts now reorganized as a sub-package of DHParser

CHANGES.txt

+DHParser Version 0.9.0 (???)
+............................
+
+-
+
+
+
 DHParser Version 0.8.9 (5.4.2019)
 .................................
 

DHParser/ebnf.py

@@ -134,11 +134,16 @@ class EBNFGrammar(Grammar):
     @ comment    = /#.*(?:\n|$)/                    # comments start with '#' and eat all chars up to and including '\n'
     @ whitespace = /\s*/                            # whitespace includes linefeed
     @ literalws  = right                            # trailing whitespace of literals will be ignored tacitly
+    @ drop       = whitespace                       # do not include whitespace in concrete syntax tree
+
+    #: top-level
 
     syntax     = [~//] { definition | directive } §EOF
     definition = symbol §"=" expression
     directive  = "@" §symbol "=" (regexp | literal | symbol) { "," (regexp | literal | symbol) }
 
+    #: components
+
     expression = term { "|" term }
     term       = { ["§"] factor }+                       # "§" means all following factors mandatory
     factor     = [flowmarker] [retrieveop] symbol !"="   # negative lookahead to be sure it's not a definition
@@ -152,21 +157,27 @@ class EBNFGrammar(Grammar):
                | repetition
                | option
 
+    #: flow-operators
+
     flowmarker = "!"  | "&"                         # '!' negative lookahead, '&' positive lookahead
                | "-!" | "-&"                        # '-' negative lookbehind, '-&' positive lookbehind
     retrieveop = "::" | ":"                         # '::' pop, ':' retrieve
 
+    #: groups
+
     group      = "(" §expression ")"
     unordered  = "<" §expression ">"                # elements of expression in arbitrary order
     oneormore  = "{" expression "}+"
     repetition = "{" §expression "}"
     option     = "[" §expression "]"
 
+    #: leaf-elements
+
     symbol     = /(?!\d)\w+/~                       # e.g. expression, factor, parameter_list
-    literal    = /"(?:[^"]|\\")*?"/~                # e.g. "(", '+', 'while'
-               | /'(?:[^']|\\')*?'/~                # whitespace following literals will be ignored tacitly.
-    plaintext  = /`(?:[^"]|\\")*?`/~                # like literal but does not eat whitespace
-    regexp     = /\/(?:\\\/|[^\/])*?\//~            # e.g. /\w+/, ~/#.*(?:\n|$)/~
+    literal    = /"(?:(?<!\\)\\"|[^"])*?"/~         # e.g. "(", '+', 'while'
+               | /'(?:(?<!\\)\\'|[^'])*?'/~         # whitespace following literals will be ignored tacitly.
+    plaintext  = /`(?:(?<!\\)\\`|[^`])*?`/~         # like literal but does not eat whitespace
+    regexp     = /\/(?:(?<!\\)\\(?:\/)|[^\/])*?\//~     # e.g. /\w+/, ~/#.*(?:\n|$)/~
     whitespace = /~/~                               # insignificant whitespace
 
     EOF = !/./

DHParser/scripts/__init__.py

+#!/usr/bin/python3
+
+"""__init__.py - package definition module for DHParser/scrips
+
+Copyright 2019  by Eckhart Arnold (arnold@badw.de)
+                Bavarian Academy of Sciences an Humanities (badw.de)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.
+"""
+

scripts/rename_project.py → DHParser/scripts/dhparser_rename.py

 #!/usr/bin/python3
 
-"""rename_project.py - rename a dhparser project properly
+"""dhparser_rename.py - rename a dhparser project properly
 
 Copyright 2019  by Eckhart Arnold (arnold@badw.de)
                 Bavarian Academy of Sciences an Humanities (badw.de)
@@ -97,4 +97,4 @@ if __name__ == "__main__":
             print(error)
             sys.exit(1)
     else:
-        print('Usage: python rename_project.py PROJECT_DIRECTORY NEW_NAME')
+        print('Usage: python dhparser_rename.py PROJECT_DIRECTORY NEW_PROJECT_NAME')

DHParser/testing.py

@@ -795,6 +795,7 @@ def runner(tests, namespace):
 
 def run_file(fname):
     if fname.lower().startswith('test_') and fname.endswith('.py'):
+        print("RUNNING " + fname)
         # print('\nRUNNING UNIT TESTS IN: ' + fname)
         exec('import ' + fname[:-3])
         runner('', eval(fname[:-3]).__dict__)

examples/Tutorial/LyrikCompiler.py

@@ -44,6 +44,7 @@ from DHParser import logging, is_filename, load_if_file, \
 def LyrikPreprocessor(text):
     return text, lambda i: i
 
+
 def get_preprocessor() -> PreprocessorFunc:
     return LyrikPreprocessor
 

examples/check_examples.py

@@ -47,8 +47,9 @@ if __name__ == "__main__":
 
     save = os.getcwd()
     os.chdir(os.path.join(scriptdir, 'Tutorial'))
-    check(os.system('python LyrikCompiler.py Lyrisches_Intermezzo_IV.txt'))
-    check(os.system('python LyrikCompiler_example.py Lyrisches_Intermezzo_IV.txt'))
+    python = 'python3 ' if os.system('python3 -V') == 0 else 'python '
+    check(os.system(python + ' LyrikCompiler.py Lyrisches_Intermezzo_IV.txt'))
+    check(os.system(python + ' LyrikCompiler_example.py Lyrisches_Intermezzo_IV.txt'))
 
     os.chdir(save)
     print()

setup.py

@@ -50,10 +50,11 @@ setup(
         'Topic :: Software Development :: Code Generators',
         'Topic :: Software Development :: Compilers'
     ],
-    scripts=['scripts/dhparser.py'],
+    scripts=['DHParser/scripts/dhparser.py',
+             'DHParser/scripts/dhparser_rename.py'],
     entry_points={
         'console_scripts': [
-            'dhparser=dhparser:main'
+            'dhparser=DHParser.scripts.dhparser:main'
         ]
     }
 )

test/run.py

@@ -35,10 +35,11 @@ def run_doctests(module):
 
 
 if __name__ == "__main__":
-    if platform.system() != "Windows":
-        interpreters = ['python3 ', 'pypy3 ']
-    else:
-        interpreters = ['python.exe ']
+    interpreters = ['python3 ' if os.system('python3 -V') == 0 else 'python ']
+    if os.system('pypy3 -V') == 0:
+        interpreters.append('pypy3 ')
+    elif os.system('pypy -V') == 0:
+        interpreters.append('pypy ')
 
     cwd = os.getcwd()
     os.chdir(os.path.join(scriptdir, '..'))
@@ -59,17 +60,17 @@ if __name__ == "__main__":
 
         # unit tests
         for interpreter in interpreters:
-            os.system(interpreter + '--version')
-            # for filename in os.listdir('test'):
-            #     if filename.startswith('test_'):
-            #         command = interpreter + os.path.join('test', filename)
-            #         results.append(pool.submit(run_unittests, command))
-            os.system(' '.join([interpreter, '-c',
-                                '''"import sys; '''
-                                '''sys.path.extend(['DHParser']);''' 
-                                '''import testing; testing.run_path('%s')"''' %
-                                scriptdir.replace('\\', '\\\\'),
-                            os.path.join(os.getcwd(), 'test')]))
+            if os.system(interpreter + '--version') == 0:
+                # for filename in os.listdir('test'):
+                #     if filename.startswith('test_'):
+                #         command = interpreter + os.path.join('test', filename)
+                #         results.append(pool.submit(run_unittests, command))
+                os.system(' '.join([interpreter, '-c',
+                                    '''"import sys; '''
+                                    '''sys.path.extend(['DHParser']);''' 
+                                    '''import testing; testing.run_path('%s')"''' %
+                                    scriptdir.replace('\\', '\\\\'),
+                                os.path.join(os.getcwd(), 'test')]))
 
         concurrent.futures.wait(results)
 

test/test_server.py

@@ -51,7 +51,7 @@ class TestServer:
             reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
             writer.write(src.encode())
             data = await reader.read(500)
-            print(f'Received: {data.decode()!r}')
+            # print(f'Received: {data.decode()!r}')
             writer.close()
 
         asyncio.run(compile('Test', ''))