Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 7c0b34d6 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- cython stringview treatmend now settled on stringview.pxd solution.

parent 25cfadc1
# This module has been shamelessly stolen from the Cython Python
# to C compiler (https://cython.org) so that DHParser does not
# cause an import error on machines where Cython is not installed
# The copyrigh to Cython is held by Greg Ewing. It is licensed
# under the Apache 2.0 licnese. (see cython.org for mor information
# cython.* namespace for pure mode.
from __future__ import absolute_import
__version__ = "0.27"
try:
from __builtin__ import basestring
except ImportError:
basestring = str
# BEGIN shameless copy from Cython/minivect/minitypes.py
class _ArrayType(object):
is_array = True
subtypes = ['dtype']
def __init__(self, dtype, ndim, is_c_contig=False, is_f_contig=False,
inner_contig=False, broadcasting=None):
self.dtype = dtype
self.ndim = ndim
self.is_c_contig = is_c_contig
self.is_f_contig = is_f_contig
self.inner_contig = inner_contig or is_c_contig or is_f_contig
self.broadcasting = broadcasting
def __repr__(self):
axes = [":"] * self.ndim
if self.is_c_contig:
axes[-1] = "::1"
elif self.is_f_contig:
axes[0] = "::1"
return "%s[%s]" % (self.dtype, ", ".join(axes))
def index_type(base_type, item):
"""
Support array type creation by slicing, e.g. double[:, :] specifies
a 2D strided array of doubles. The syntax is the same as for
Cython memoryviews.
"""
class InvalidTypeSpecification(Exception):
pass
def verify_slice(s):
if s.start or s.stop or s.step not in (None, 1):
raise InvalidTypeSpecification(
"Only a step of 1 may be provided to indicate C or "
"Fortran contiguity")
if isinstance(item, tuple):
step_idx = None
for idx, s in enumerate(item):
verify_slice(s)
if s.step and (step_idx or idx not in (0, len(item) - 1)):
raise InvalidTypeSpecification(
"Step may only be provided once, and only in the "
"first or last dimension.")
if s.step == 1:
step_idx = idx
return _ArrayType(base_type, len(item),
is_c_contig=step_idx == len(item) - 1,
is_f_contig=step_idx == 0)
elif isinstance(item, slice):
verify_slice(item)
return _ArrayType(base_type, 1, is_c_contig=bool(item.step))
else:
# int[8] etc.
assert int(item) == item # array size must be a plain integer
array(base_type, item)
# END shameless copy
compiled = False
_Unspecified = object()
# Function decorators
def _empty_decorator(x):
return x
def locals(**arg_types):
return _empty_decorator
def test_assert_path_exists(*paths):
return _empty_decorator
def test_fail_if_path_exists(*paths):
return _empty_decorator
class _EmptyDecoratorAndManager(object):
def __call__(self, x):
return x
def __enter__(self):
pass
def __exit__(self, exc_type, exc_value, traceback):
pass
class _Optimization(object):
pass
cclass = ccall = cfunc = _EmptyDecoratorAndManager()
returns = wraparound = boundscheck = initializedcheck = nonecheck = \
overflowcheck = embedsignature = cdivision = cdivision_warnings = \
always_allows_keywords = profile = linetrace = infer_type = \
unraisable_tracebacks = freelist = \
lambda _: _EmptyDecoratorAndManager()
exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager()
optimization = _Optimization()
overflowcheck.fold = optimization.use_switch = \
optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager()
final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator
_cython_inline = None
def inline(f, *args, **kwds):
if isinstance(f, basestring):
global _cython_inline
if _cython_inline is None:
from Cython.Build.Inline import cython_inline as _cython_inline
return _cython_inline(f, *args, **kwds)
else:
assert len(args) == len(kwds) == 0
return f
def compile(f):
from Cython.Build.Inline import RuntimeCompiledFunction
return RuntimeCompiledFunction(f)
# Special functions
def cdiv(a, b):
q = a / b
if q < 0:
q += 1
return q
def cmod(a, b):
r = a % b
if (a * b) < 0:
r -= b
return r
# Emulated language constructs
def cast(type, *args, **kwargs):
kwargs.pop('typecheck', None)
assert not kwargs
if hasattr(type, '__call__'):
return type(*args)
else:
return args[0]
def sizeof(arg):
return 1
def typeof(arg):
return arg.__class__.__name__
# return type(arg)
def address(arg):
return pointer(type(arg))([arg])
def declare(type=None, value=_Unspecified, **kwds):
if type not in (None, object) and hasattr(type, '__call__'):
if value is not _Unspecified:
return type(value)
else:
return type()
else:
return value
class _nogil(object):
"""Support for 'with nogil' statement
"""
def __enter__(self):
pass
def __exit__(self, exc_class, exc, tb):
return exc_class is None
nogil = _nogil()
gil = _nogil()
del _nogil
# Emulated types
class CythonMetaType(type):
def __getitem__(type, ix):
return array(type, ix)
CythonTypeObject = CythonMetaType('CythonTypeObject', (object,), {})
class CythonType(CythonTypeObject):
def _pointer(self, n=1):
for i in range(n):
self = pointer(self)
return self
class PointerType(CythonType):
def __init__(self, value=None):
if isinstance(value, (ArrayType, PointerType)):
self._items = [cast(self._basetype, a) for a in value._items]
elif isinstance(value, list):
self._items = [cast(self._basetype, a) for a in value]
elif value is None or value == 0:
self._items = []
else:
raise ValueError
def __getitem__(self, ix):
if ix < 0:
raise IndexError("negative indexing not allowed in C")
return self._items[ix]
def __setitem__(self, ix, value):
if ix < 0:
raise IndexError("negative indexing not allowed in C")
self._items[ix] = cast(self._basetype, value)
def __eq__(self, value):
if value is None and not self._items:
return True
elif type(self) != type(value):
return False
else:
return not self._items and not value._items
def __repr__(self):
return "%s *" % (self._basetype,)
class ArrayType(PointerType):
def __init__(self):
self._items = [None] * self._n
class StructType(CythonType):
def __init__(self, cast_from=_Unspecified, **data):
if cast_from is not _Unspecified:
# do cast
if len(data) > 0:
raise ValueError('Cannot accept keyword arguments when casting.')
if type(cast_from) is not type(self):
raise ValueError('Cannot cast from %s' % cast_from)
for key, value in cast_from.__dict__.items():
setattr(self, key, value)
else:
for key, value in data.items():
setattr(self, key, value)
def __setattr__(self, key, value):
if key in self._members:
self.__dict__[key] = cast(self._members[key], value)
else:
raise AttributeError("Struct has no member '%s'" % key)
class UnionType(CythonType):
def __init__(self, cast_from=_Unspecified, **data):
if cast_from is not _Unspecified:
# do type cast
if len(data) > 0:
raise ValueError('Cannot accept keyword arguments when casting.')
if isinstance(cast_from, dict):
datadict = cast_from
elif type(cast_from) is type(self):
datadict = cast_from.__dict__
else:
raise ValueError('Cannot cast from %s' % cast_from)
else:
datadict = data
if len(datadict) > 1:
raise AttributeError("Union can only store one field at a time.")
for key, value in datadict.items():
setattr(self, key, value)
def __setattr__(self, key, value):
if key in '__dict__':
CythonType.__setattr__(self, key, value)
elif key in self._members:
self.__dict__ = {key: cast(self._members[key], value)}
else:
raise AttributeError("Union has no member '%s'" % key)
def pointer(basetype):
class PointerInstance(PointerType):
_basetype = basetype
return PointerInstance
def array(basetype, n):
class ArrayInstance(ArrayType):
_basetype = basetype
_n = n
return ArrayInstance
def struct(**members):
class StructInstance(StructType):
_members = members
for key in members:
setattr(StructInstance, key, None)
return StructInstance
def union(**members):
class UnionInstance(UnionType):
_members = members
for key in members:
setattr(UnionInstance, key, None)
return UnionInstance
class typedef(CythonType):
def __init__(self, type, name=None):
self._basetype = type
self.name = name
def __call__(self, *arg):
value = cast(self._basetype, *arg)
return value
def __repr__(self):
return self.name or str(self._basetype)
__getitem__ = index_type
class _FusedType(CythonType):
pass
def fused_type(*args):
if not args:
raise TypeError("Expected at least one type as argument")
# Find the numeric type with biggest rank if all types are numeric
rank = -1
for type in args:
if type not in (py_int, py_long, py_float, py_complex):
break
if type_ordering.index(type) > rank:
result_type = type
else:
return result_type
# Not a simple numeric type, return a fused type instance. The result
# isn't really meant to be used, as we can't keep track of the context in
# pure-mode. Casting won't do anything in this case.
return _FusedType()
def _specialized_from_args(signatures, args, kwargs):
"Perhaps this should be implemented in a TreeFragment in Cython code"
raise Exception("yet to be implemented")
py_int = typedef(int, "int")
try:
py_long = typedef(long, "long")
except NameError: # Py3
py_long = typedef(int, "long")
py_float = typedef(float, "float")
py_complex = typedef(complex, "double complex")
# Predefined types
int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t',
'size_t']
float_types = ['longdouble', 'double', 'float']
complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex']
other_types = ['bint', 'void']
to_repr = {
'longlong': 'long long',
'longdouble': 'long double',
'longdoublecomplex': 'long double complex',
'doublecomplex': 'double complex',
'floatcomplex': 'float complex',
}.get
gs = globals()
# note: cannot simply name the unicode type here as 2to3 gets in the way and replaces it by str
try:
import __builtin__ as builtins
except ImportError: # Py3
import builtins
gs['unicode'] = typedef(getattr(builtins, 'unicode', str), 'unicode')
del builtins
for name in int_types:
reprname = to_repr(name, name)
gs[name] = typedef(py_int, reprname)
if name not in ('Py_UNICODE', 'Py_UCS4') and not name.endswith('size_t'):
gs['u' + name] = typedef(py_int, "unsigned " + reprname)
gs['s' + name] = typedef(py_int, "signed " + reprname)
for name in float_types:
gs[name] = typedef(py_float, to_repr(name, name))
for name in complex_types:
gs[name] = typedef(py_complex, to_repr(name, name))
bint = typedef(bool, "bint")
void = typedef(int, "void")
for t in int_types + float_types + complex_types + other_types:
for i in range(1, 4):
gs["%s_%s" % ('p' * i, t)] = gs[t]._pointer(i)
void = typedef(None, "void")
NULL = gs['p_void'](0)
# looks like 'gs' has some users out there by now...
# del gs
integral = floating = numeric = _FusedType()
type_ordering = [py_int, py_long, py_float, py_complex]
class CythonDotParallel(object):
"""
The cython.parallel module.
"""
__all__ = ['parallel', 'prange', 'threadid']
def parallel(self, num_threads=None):
return nogil
def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False):
if stop is None:
stop = start
start = 0
return range(start, stop, step)
def threadid(self):
return 0
# def threadsavailable(self):
# return 1
import sys
sys.modules['cython.parallel'] = CythonDotParallel()
del sys
......@@ -20,17 +20,16 @@ compilation of domain specific languages based on an EBNF-grammar.
"""
import os
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
re, typing
from DHParser.error import Error, is_error, has_errors, only_errors
from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, TransformationFunc
from DHParser.error import Error, is_error, has_errors, only_errors
from typing import Any, cast, Tuple, Union, Iterator, Iterable
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
re
__all__ = ('GrammarError',
'CompilationError',
......@@ -170,7 +169,7 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
# read grammar
grammar_src = load_if_file(grammar_representation)
if is_python_code(grammar_src):
parser_py, messages, AST = grammar_src, [], None
parser_py, messages, AST = grammar_src, [], None # type: str, List[Error], Node
else:
with logging(False):
parser_py, messages, AST = compile_source(grammar_src, None,
......@@ -371,7 +370,7 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
"""
Compiles the a source file with a given compiler and writes the
result to a file.
......@@ -416,7 +415,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
cfactory = get_ebnf_compiler
compiler1 = cfactory()
compiler1.set_grammar_name(compiler_name, source_file)
result, messages, ast = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
if has_errors(messages):
return messages
......@@ -522,7 +521,7 @@ def recompile_grammar(ebnf_filename, force=False) -> bool:
base, ext = os.path.splitext(ebnf_filename)
compiler_name = base + 'Compiler.py'
error_file_name = base + '_ebnf_ERRORS.txt'
messages = [] # type: Iterable[str]
messages = [] # type: Iterable[Error]
if (not os.path.exists(compiler_name) or force or
grammar_changed(compiler_name, ebnf_filename)):
# print("recompiling parser for: " + ebnf_filename)
......
......@@ -44,7 +44,7 @@ class Error:
MANDATORY_CONTINUATION = 1001
def __init__(self, message: str, level: int = ERROR, code: Hashable = 0,
pos: int = -1, line: int = -1, column: int = -1):
pos: int = -1, line: int = -1, column: int = -1) -> None:
self.message = message
assert level >= 0
self.level = level or Error.ERROR
......
......@@ -60,14 +60,14 @@ import abc
import copy
import os
from functools import partial
from typing import Any, Callable, cast, Dict, List, Set, Tuple, Union
from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, load_if_file, \
re, typing
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, TOKEN_PTYPE, \
ZOMBIE_PARSER
from DHParser.error import Error, is_error, has_errors, linebreaks, line_col
from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union, Optional
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_PARSER
from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, \
load_if_file, re
__all__ = ('PreprocessorFunc',
'HistoryRecord',
......@@ -690,7 +690,6 @@ class Grammar:
parser.parser._name = entry
cls.parser_initialization__ = "done"
def __init__(self, root: Parser=None) -> None:
# if not hasattr(self.__class__, 'parser_initialization__'):
# self.__class__.parser_initialization__ = "pending"
......@@ -1603,7 +1602,7 @@ class Lookbehind(FlowOperator):
while isinstance(p, Synonym):
p = p.parser
assert isinstance(p, RegExp), str(type(p))
self.regexp = p.main.regexp if isinstance(p, RE) else p.regexp
self.regexp = cast(RE, p).main.regexp if isinstance(p, RE) else p.regexp
super(Lookbehind, self).__init__(parser, name)
def __call__(self, text: StringView) -> Tuple[Node, StringView]:
......@@ -1941,10 +1940,10 @@ class Compiler:
def compile_source(source: str,
preprocessor: PreprocessorFunc, # str -> str
parser: Grammar, # str -> Node (concrete syntax tree (CST))