Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit c4a8107c authored by eckhart's avatar eckhart
Browse files

- unit tests can now be run concurrently

parent 36592546
......@@ -24,9 +24,10 @@ so that they are best defined in a toolkit-module.
import codecs
import hashlib
import inspect
import io
import multiprocessing
import parser
import threading
try:
import regex as re
......@@ -46,6 +47,7 @@ from typing import Any, Iterable, Sequence, Set, Union, Dict, cast
__all__ = ('escape_re',
'escape_control_characters',
'is_filename',
'concurrent_ident',
'lstrip_docstring',
'issubtype',
'isgenerictype',
......@@ -111,6 +113,13 @@ def is_filename(strg: str) -> bool:
# and strg.select('*') < 0 and strg.select('?') < 0
def concurrent_ident() -> str:
"""
Returns an identificator for the current process and thread
"""
return multiprocessing.current_process().name + '_' + str(threading.get_ident())
#######################################################################
#
# type system support
......
......@@ -3,27 +3,19 @@
"""Runs the dhparser test-suite with several installed interpreters"""
import concurrent.futures
import multiprocessing
import os
import platform
import time
def run_tests(testtype, command):
def run_tests(command):
testtype = 'DOCTEST' if command.find('doctest') >= 0 else 'UNITTEST'
filename = command[command.rfind(' ')+1:]
print('\n' + testtype + ' ' + filename)
os.system(command)
def run_unittests(command):
run_tests('UNITTESTS', command)
def run_doctests(command):
print(os.getcwd())
run_tests('DOCTESTS', command)
if __name__ == "__main__":
scriptdir = os.path.dirname(os.path.realpath(__file__))
......@@ -32,7 +24,7 @@ if __name__ == "__main__":
# print("Running nosetests:")
# os.system("nosetests test")
if platform.system() != "Windows":
interpreters = ['python3 ', 'pypy3 ']
interpreters = ['pypy3 ', 'python3 ']
else:
interpreters = ['python.exe ']
......@@ -41,22 +33,20 @@ if __name__ == "__main__":
timestamp = time.time()
with concurrent.futures.ThreadPoolExecutor(4) as pool:
with concurrent.futures.ProcessPoolExecutor(multiprocessing.cpu_count()) as pool:
for interpreter in interpreters:
os.system(interpreter + '--version')
# unit tests
commands = [interpreter + os.path.join('test', filename)
for filename in os.listdir('test') if filename.startswith('test_')]
pool.map(run_unittests, commands)
# doctests
commands = [interpreter + ' -m doctest ' + os.path.join('DHParser', filename)
for filename in os.listdir('DHParser') if filename.endswith('.py')
and filename not in ["foreign_typing.py", "stringview.py", "__init__.py"]]
pool.map(run_doctests, commands)
# unit tests
commands += [interpreter + os.path.join('test', filename)
for filename in os.listdir('test') if filename.startswith('test_')]
pool.map(run_tests, commands)
elapsed = time.time() - timestamp
print('\n Test-Duration: %.2f seconds' % elapsed)
......
......@@ -29,6 +29,7 @@ from DHParser import Compiler
from DHParser.error import is_error
from DHParser.dsl import compile_on_disk, run_compiler, compileEBNF, grammar_provider, \
load_compiler_suite
from DHParser.toolkit import concurrent_ident
ARITHMETIC_EBNF = """
@ whitespace = linefeed
......@@ -62,12 +63,13 @@ class TestCompilerGeneration:
word = /\w+/
WSPC = /\s+/
"""
tmp = 'test/tmp/' if os.path.isdir('test/') else ('tmp/')
tmpname = 'tmp_' + concurrent_ident()
tmp = os.path.join('test', tmpname) if os.path.isdir('test/') else tmpname
trivial_text = u"""Es war ein Koenig in Thule.\n"""
grammar_name = tmp + "TestCompilerGeneration.ebnf"
compiler_name = tmp + "TestCompilerGenerationCompiler.py"
text_name = tmp + "TestCompilerGeneration_text.txt"
result_name = tmp + "TestCompilerGeneration_text.xml"
grammar_name = os.path.join(tmp, "TestCompilerGeneration.ebnf")
compiler_name = os.path.join(tmp, "TestCompilerGenerationCompiler.py")
text_name = os.path.join(tmp, "TestCompilerGeneration_text.txt")
result_name = os.path.join(tmp, "TestCompilerGeneration_text.xml")
def setup(self):
if not os.path.exists(self.tmp):
......
......@@ -28,12 +28,14 @@ import sys
sys.path.extend(['../', './'])
from DHParser.toolkit import has_fenced_code, load_if_file, re, \
lstrip_docstring, issubtype, isgenerictype, typing
lstrip_docstring, issubtype, typing, concurrent_ident
from DHParser.log import log_dir, logging, is_logging
class TestLoggingAndLoading:
filename = "test/tmp/test.py" if os.path.isdir('test') else "tmp/test.py"
tmpname = 'tmp_' + concurrent_ident()
filename = os.path.join("test", tmpname, "test.py") if os.path.isdir('test') \
else os.path.join(tmpname, "test.py")
dirname = os.path.dirname(filename)
code1 = "x = 46"
code2 = "def f():\n return 46"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment