- unit tests can now be run concurrently

DHParser/toolkit.py

@@ -24,9 +24,10 @@ so that they are best defined in a toolkit-module.
 
 import codecs
 import hashlib
-import inspect
 import io
+import multiprocessing
 import parser
+import threading
 
 try:
     import regex as re
@@ -46,6 +47,7 @@ from typing import Any, Iterable, Sequence, Set, Union, Dict, cast
 __all__ = ('escape_re',
            'escape_control_characters',
            'is_filename',
+           'concurrent_ident',
            'lstrip_docstring',
            'issubtype',
            'isgenerictype',
@@ -111,6 +113,13 @@ def is_filename(strg: str) -> bool:
     #   and strg.select('*') < 0 and strg.select('?') < 0
 
 
+def concurrent_ident() -> str:
+    """
+    Returns an identificator for the current process and thread
+    """
+    return multiprocessing.current_process().name + '_' + str(threading.get_ident())
+
+
 #######################################################################
 #
 # type system support

test/run.py

@@ -3,27 +3,19 @@
 """Runs the dhparser test-suite with several installed interpreters"""
 
 import concurrent.futures
+import multiprocessing
 import os
 import platform
 import time
 
 
-
-def run_tests(testtype, command):
+def run_tests(command):
+    testtype = 'DOCTEST' if command.find('doctest') >= 0 else 'UNITTEST'
     filename = command[command.rfind(' ')+1:]
     print('\n' + testtype + ' ' + filename)
     os.system(command)
 
 
-def run_unittests(command):
-    run_tests('UNITTESTS', command)
-
-
-def run_doctests(command):
-    print(os.getcwd())
-    run_tests('DOCTESTS', command)
-
-
 if __name__ == "__main__":
     scriptdir = os.path.dirname(os.path.realpath(__file__))
 
@@ -32,7 +24,7 @@ if __name__ == "__main__":
     # print("Running nosetests:")
     # os.system("nosetests test")
     if platform.system() != "Windows":
-        interpreters = ['python3 ', 'pypy3 ']
+        interpreters = ['pypy3 ', 'python3 ']
     else:
         interpreters = ['python.exe ']
 
@@ -41,22 +33,20 @@ if __name__ == "__main__":
 
     timestamp = time.time()
 
-    with concurrent.futures.ThreadPoolExecutor(4) as pool:
+    with concurrent.futures.ProcessPoolExecutor(multiprocessing.cpu_count()) as pool:
         for interpreter in interpreters:
             os.system(interpreter + '--version')
 
-            # unit tests
-
-            commands = [interpreter + os.path.join('test', filename)
-                        for filename in os.listdir('test') if filename.startswith('test_')]
-            pool.map(run_unittests, commands)
-
             # doctests
-
             commands = [interpreter + ' -m doctest ' + os.path.join('DHParser', filename)
                         for filename in os.listdir('DHParser') if filename.endswith('.py')
                         and filename not in ["foreign_typing.py", "stringview.py", "__init__.py"]]
-            pool.map(run_doctests, commands)
+
+            # unit tests
+            commands += [interpreter + os.path.join('test', filename)
+                         for filename in os.listdir('test') if filename.startswith('test_')]
+
+            pool.map(run_tests, commands)
 
     elapsed = time.time() - timestamp
     print('\n Test-Duration: %.2f seconds' % elapsed)

test/test_dsl.py

@@ -29,6 +29,7 @@ from DHParser import Compiler
 from DHParser.error import is_error
 from DHParser.dsl import compile_on_disk, run_compiler, compileEBNF, grammar_provider, \
     load_compiler_suite
+from DHParser.toolkit import concurrent_ident
 
 ARITHMETIC_EBNF = """
     @ whitespace = linefeed
@@ -62,12 +63,13 @@ class TestCompilerGeneration:
         word = /\w+/
         WSPC = /\s+/
         """
-    tmp = 'test/tmp/' if os.path.isdir('test/') else ('tmp/')
+    tmpname = 'tmp_' + concurrent_ident()
+    tmp = os.path.join('test', tmpname) if os.path.isdir('test/') else tmpname
     trivial_text = u"""Es war ein Koenig in Thule.\n"""
-    grammar_name = tmp + "TestCompilerGeneration.ebnf"
-    compiler_name = tmp + "TestCompilerGenerationCompiler.py"
-    text_name = tmp + "TestCompilerGeneration_text.txt"
-    result_name = tmp + "TestCompilerGeneration_text.xml"
+    grammar_name = os.path.join(tmp, "TestCompilerGeneration.ebnf")
+    compiler_name = os.path.join(tmp, "TestCompilerGenerationCompiler.py")
+    text_name = os.path.join(tmp, "TestCompilerGeneration_text.txt")
+    result_name = os.path.join(tmp, "TestCompilerGeneration_text.xml")
 
     def setup(self):
         if not os.path.exists(self.tmp):

test/test_toolkit.py

@@ -28,12 +28,14 @@ import sys
 sys.path.extend(['../', './'])
 
 from DHParser.toolkit import has_fenced_code, load_if_file, re, \
-    lstrip_docstring, issubtype, isgenerictype, typing
+    lstrip_docstring, issubtype, typing, concurrent_ident
 from DHParser.log import log_dir, logging, is_logging
 
 
 class TestLoggingAndLoading:
-    filename = "test/tmp/test.py" if os.path.isdir('test') else "tmp/test.py"
+    tmpname = 'tmp_' + concurrent_ident()
+    filename = os.path.join("test", tmpname, "test.py") if os.path.isdir('test') \
+        else os.path.join(tmpname, "test.py")
     dirname = os.path.dirname(filename)
     code1 = "x = 46"
     code2 = "def f():\n    return 46"