- cython stringview treatmend now settled on stringview.pxd solution.

DHParser/VERALTET/foreign_cython.py

+# This module has been shamelessly stolen from the Cython Python
+# to C compiler (https://cython.org) so that DHParser does not
+# cause an import error on machines where Cython is not installed
+# The copyrigh to Cython is held by Greg Ewing. It is licensed
+# under the Apache 2.0 licnese. (see cython.org for mor information
+
+# cython.* namespace for pure mode.
+from __future__ import absolute_import
+
+__version__ = "0.27"
+
+try:
+    from __builtin__ import basestring
+except ImportError:
+    basestring = str
+
+
+# BEGIN shameless copy from Cython/minivect/minitypes.py
+
+class _ArrayType(object):
+    is_array = True
+    subtypes = ['dtype']
+
+    def __init__(self, dtype, ndim, is_c_contig=False, is_f_contig=False,
+                 inner_contig=False, broadcasting=None):
+        self.dtype = dtype
+        self.ndim = ndim
+        self.is_c_contig = is_c_contig
+        self.is_f_contig = is_f_contig
+        self.inner_contig = inner_contig or is_c_contig or is_f_contig
+        self.broadcasting = broadcasting
+
+    def __repr__(self):
+        axes = [":"] * self.ndim
+        if self.is_c_contig:
+            axes[-1] = "::1"
+        elif self.is_f_contig:
+            axes[0] = "::1"
+
+        return "%s[%s]" % (self.dtype, ", ".join(axes))
+
+
+def index_type(base_type, item):
+    """
+    Support array type creation by slicing, e.g. double[:, :] specifies
+    a 2D strided array of doubles. The syntax is the same as for
+    Cython memoryviews.
+    """
+
+    class InvalidTypeSpecification(Exception):
+        pass
+
+    def verify_slice(s):
+        if s.start or s.stop or s.step not in (None, 1):
+            raise InvalidTypeSpecification(
+                "Only a step of 1 may be provided to indicate C or "
+                "Fortran contiguity")
+
+    if isinstance(item, tuple):
+        step_idx = None
+        for idx, s in enumerate(item):
+            verify_slice(s)
+            if s.step and (step_idx or idx not in (0, len(item) - 1)):
+                raise InvalidTypeSpecification(
+                    "Step may only be provided once, and only in the "
+                    "first or last dimension.")
+
+            if s.step == 1:
+                step_idx = idx
+
+        return _ArrayType(base_type, len(item),
+                          is_c_contig=step_idx == len(item) - 1,
+                          is_f_contig=step_idx == 0)
+    elif isinstance(item, slice):
+        verify_slice(item)
+        return _ArrayType(base_type, 1, is_c_contig=bool(item.step))
+    else:
+        # int[8] etc.
+        assert int(item) == item  # array size must be a plain integer
+        array(base_type, item)
+
+
+# END shameless copy
+
+
+compiled = False
+
+_Unspecified = object()
+
+
+# Function decorators
+
+def _empty_decorator(x):
+    return x
+
+
+def locals(**arg_types):
+    return _empty_decorator
+
+
+def test_assert_path_exists(*paths):
+    return _empty_decorator
+
+
+def test_fail_if_path_exists(*paths):
+    return _empty_decorator
+
+
+class _EmptyDecoratorAndManager(object):
+    def __call__(self, x):
+        return x
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+
+class _Optimization(object):
+    pass
+
+
+cclass = ccall = cfunc = _EmptyDecoratorAndManager()
+
+returns = wraparound = boundscheck = initializedcheck = nonecheck = \
+    overflowcheck = embedsignature = cdivision = cdivision_warnings = \
+    always_allows_keywords = profile = linetrace = infer_type = \
+    unraisable_tracebacks = freelist = \
+    lambda _: _EmptyDecoratorAndManager()
+
+exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager()
+
+optimization = _Optimization()
+
+overflowcheck.fold = optimization.use_switch = \
+    optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager()
+
+final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator
+
+_cython_inline = None
+
+
+def inline(f, *args, **kwds):
+    if isinstance(f, basestring):
+        global _cython_inline
+        if _cython_inline is None:
+            from Cython.Build.Inline import cython_inline as _cython_inline
+        return _cython_inline(f, *args, **kwds)
+    else:
+        assert len(args) == len(kwds) == 0
+        return f
+
+
+def compile(f):
+    from Cython.Build.Inline import RuntimeCompiledFunction
+    return RuntimeCompiledFunction(f)
+
+
+# Special functions
+
+def cdiv(a, b):
+    q = a / b
+    if q < 0:
+        q += 1
+    return q
+
+
+def cmod(a, b):
+    r = a % b
+    if (a * b) < 0:
+        r -= b
+    return r
+
+
+# Emulated language constructs
+
+def cast(type, *args, **kwargs):
+    kwargs.pop('typecheck', None)
+    assert not kwargs
+    if hasattr(type, '__call__'):
+        return type(*args)
+    else:
+        return args[0]
+
+
+def sizeof(arg):
+    return 1
+
+
+def typeof(arg):
+    return arg.__class__.__name__
+    # return type(arg)
+
+
+def address(arg):
+    return pointer(type(arg))([arg])
+
+
+def declare(type=None, value=_Unspecified, **kwds):
+    if type not in (None, object) and hasattr(type, '__call__'):
+        if value is not _Unspecified:
+            return type(value)
+        else:
+            return type()
+    else:
+        return value
+
+
+class _nogil(object):
+    """Support for 'with nogil' statement
+    """
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_class, exc, tb):
+        return exc_class is None
+
+
+nogil = _nogil()
+gil = _nogil()
+del _nogil
+
+
+# Emulated types
+
+class CythonMetaType(type):
+    def __getitem__(type, ix):
+        return array(type, ix)
+
+
+CythonTypeObject = CythonMetaType('CythonTypeObject', (object,), {})
+
+
+class CythonType(CythonTypeObject):
+    def _pointer(self, n=1):
+        for i in range(n):
+            self = pointer(self)
+        return self
+
+
+class PointerType(CythonType):
+    def __init__(self, value=None):
+        if isinstance(value, (ArrayType, PointerType)):
+            self._items = [cast(self._basetype, a) for a in value._items]
+        elif isinstance(value, list):
+            self._items = [cast(self._basetype, a) for a in value]
+        elif value is None or value == 0:
+            self._items = []
+        else:
+            raise ValueError
+
+    def __getitem__(self, ix):
+        if ix < 0:
+            raise IndexError("negative indexing not allowed in C")
+        return self._items[ix]
+
+    def __setitem__(self, ix, value):
+        if ix < 0:
+            raise IndexError("negative indexing not allowed in C")
+        self._items[ix] = cast(self._basetype, value)
+
+    def __eq__(self, value):
+        if value is None and not self._items:
+            return True
+        elif type(self) != type(value):
+            return False
+        else:
+            return not self._items and not value._items
+
+    def __repr__(self):
+        return "%s *" % (self._basetype,)
+
+
+class ArrayType(PointerType):
+    def __init__(self):
+        self._items = [None] * self._n
+
+
+class StructType(CythonType):
+    def __init__(self, cast_from=_Unspecified, **data):
+        if cast_from is not _Unspecified:
+            # do cast
+            if len(data) > 0:
+                raise ValueError('Cannot accept keyword arguments when casting.')
+            if type(cast_from) is not type(self):
+                raise ValueError('Cannot cast from %s' % cast_from)
+            for key, value in cast_from.__dict__.items():
+                setattr(self, key, value)
+        else:
+            for key, value in data.items():
+                setattr(self, key, value)
+
+    def __setattr__(self, key, value):
+        if key in self._members:
+            self.__dict__[key] = cast(self._members[key], value)
+        else:
+            raise AttributeError("Struct has no member '%s'" % key)
+
+
+class UnionType(CythonType):
+    def __init__(self, cast_from=_Unspecified, **data):
+        if cast_from is not _Unspecified:
+            # do type cast
+            if len(data) > 0:
+                raise ValueError('Cannot accept keyword arguments when casting.')
+            if isinstance(cast_from, dict):
+                datadict = cast_from
+            elif type(cast_from) is type(self):
+                datadict = cast_from.__dict__
+            else:
+                raise ValueError('Cannot cast from %s' % cast_from)
+        else:
+            datadict = data
+        if len(datadict) > 1:
+            raise AttributeError("Union can only store one field at a time.")
+        for key, value in datadict.items():
+            setattr(self, key, value)
+
+    def __setattr__(self, key, value):
+        if key in '__dict__':
+            CythonType.__setattr__(self, key, value)
+        elif key in self._members:
+            self.__dict__ = {key: cast(self._members[key], value)}
+        else:
+            raise AttributeError("Union has no member '%s'" % key)
+
+
+def pointer(basetype):
+    class PointerInstance(PointerType):
+        _basetype = basetype
+
+    return PointerInstance
+
+
+def array(basetype, n):
+    class ArrayInstance(ArrayType):
+        _basetype = basetype
+        _n = n
+
+    return ArrayInstance
+
+
+def struct(**members):
+    class StructInstance(StructType):
+        _members = members
+
+    for key in members:
+        setattr(StructInstance, key, None)
+    return StructInstance
+
+
+def union(**members):
+    class UnionInstance(UnionType):
+        _members = members
+
+    for key in members:
+        setattr(UnionInstance, key, None)
+    return UnionInstance
+
+
+class typedef(CythonType):
+    def __init__(self, type, name=None):
+        self._basetype = type
+        self.name = name
+
+    def __call__(self, *arg):
+        value = cast(self._basetype, *arg)
+        return value
+
+    def __repr__(self):
+        return self.name or str(self._basetype)
+
+    __getitem__ = index_type
+
+
+class _FusedType(CythonType):
+    pass
+
+
+def fused_type(*args):
+    if not args:
+        raise TypeError("Expected at least one type as argument")
+
+    # Find the numeric type with biggest rank if all types are numeric
+    rank = -1
+    for type in args:
+        if type not in (py_int, py_long, py_float, py_complex):
+            break
+
+        if type_ordering.index(type) > rank:
+            result_type = type
+    else:
+        return result_type
+
+    # Not a simple numeric type, return a fused type instance. The result
+    # isn't really meant to be used, as we can't keep track of the context in
+    # pure-mode. Casting won't do anything in this case.
+    return _FusedType()
+
+
+def _specialized_from_args(signatures, args, kwargs):
+    "Perhaps this should be implemented in a TreeFragment in Cython code"
+    raise Exception("yet to be implemented")
+
+
+py_int = typedef(int, "int")
+try:
+    py_long = typedef(long, "long")
+except NameError:  # Py3
+    py_long = typedef(int, "long")
+py_float = typedef(float, "float")
+py_complex = typedef(complex, "double complex")
+
+# Predefined types
+
+int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t',
+             'size_t']
+float_types = ['longdouble', 'double', 'float']
+complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex']
+other_types = ['bint', 'void']
+
+to_repr = {
+    'longlong': 'long long',
+    'longdouble': 'long double',
+    'longdoublecomplex': 'long double complex',
+    'doublecomplex': 'double complex',
+    'floatcomplex': 'float complex',
+}.get
+
+gs = globals()
+
+# note: cannot simply name the unicode type here as 2to3 gets in the way and replaces it by str
+try:
+    import __builtin__ as builtins
+except ImportError:  # Py3
+    import builtins
+
+gs['unicode'] = typedef(getattr(builtins, 'unicode', str), 'unicode')
+del builtins
+
+for name in int_types:
+    reprname = to_repr(name, name)
+    gs[name] = typedef(py_int, reprname)
+    if name not in ('Py_UNICODE', 'Py_UCS4') and not name.endswith('size_t'):
+        gs['u' + name] = typedef(py_int, "unsigned " + reprname)
+        gs['s' + name] = typedef(py_int, "signed " + reprname)
+
+for name in float_types:
+    gs[name] = typedef(py_float, to_repr(name, name))
+
+for name in complex_types:
+    gs[name] = typedef(py_complex, to_repr(name, name))
+
+bint = typedef(bool, "bint")
+void = typedef(int, "void")
+
+for t in int_types + float_types + complex_types + other_types:
+    for i in range(1, 4):
+        gs["%s_%s" % ('p' * i, t)] = gs[t]._pointer(i)
+
+void = typedef(None, "void")
+NULL = gs['p_void'](0)
+
+# looks like 'gs' has some users out there by now...
+# del gs
+
+integral = floating = numeric = _FusedType()
+
+type_ordering = [py_int, py_long, py_float, py_complex]
+
+
+class CythonDotParallel(object):
+    """
+    The cython.parallel module.
+    """
+
+    __all__ = ['parallel', 'prange', 'threadid']
+
+    def parallel(self, num_threads=None):
+        return nogil
+
+    def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False):
+        if stop is None:
+            stop = start
+            start = 0
+        return range(start, stop, step)
+
+    def threadid(self):
+        return 0
+
+        # def threadsavailable(self):
+        # return 1
+
+
+import sys
+
+sys.modules['cython.parallel'] = CythonDotParallel()
+del sys

DHParser/dsl.py

@@ -20,17 +20,16 @@ compilation of domain specific languages based on an EBNF-grammar.
 """
 
 import os
+from typing import Any, cast, List, Tuple, Union, Iterator, Iterable
 
 from DHParser.ebnf import EBNFCompiler, grammar_changed, \
     get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
     PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
-from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
-    re, typing
+from DHParser.error import Error, is_error, has_errors, only_errors
 from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
 from DHParser.syntaxtree import Node, TransformationFunc
-from DHParser.error import Error, is_error, has_errors, only_errors
-
-from typing import Any, cast, Tuple, Union, Iterator, Iterable
+from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
+    re
 
 __all__ = ('GrammarError',
            'CompilationError',
@@ -170,7 +169,7 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
         # read grammar
         grammar_src = load_if_file(grammar_representation)
         if is_python_code(grammar_src):
-            parser_py, messages, AST = grammar_src, [], None
+            parser_py, messages, AST = grammar_src, [], None  # type: str, List[Error], Node
         else:
             with logging(False):
                 parser_py, messages, AST = compile_source(grammar_src, None,
@@ -371,7 +370,7 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
     return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
 
 
-def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
+def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
     """
     Compiles the a source file with a given compiler and writes the
     result to a file.
@@ -416,7 +415,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
         cfactory = get_ebnf_compiler
     compiler1 = cfactory()
     compiler1.set_grammar_name(compiler_name, source_file)
-    result, messages, ast = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
+    result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
     if has_errors(messages):
         return messages
 
@@ -522,7 +521,7 @@ def recompile_grammar(ebnf_filename, force=False) -> bool:
     base, ext = os.path.splitext(ebnf_filename)
     compiler_name = base + 'Compiler.py'
     error_file_name = base + '_ebnf_ERRORS.txt'
-    messages = []  # type: Iterable[str]
+    messages = []  # type: Iterable[Error]
     if (not os.path.exists(compiler_name) or force or
             grammar_changed(compiler_name, ebnf_filename)):
         # print("recompiling parser for: " + ebnf_filename)

DHParser/error.py

@@ -44,7 +44,7 @@ class Error:
     MANDATORY_CONTINUATION = 1001
 
     def __init__(self, message: str, level: int = ERROR, code: Hashable = 0,
-                 pos: int = -1, line: int = -1, column: int = -1):
+                 pos: int = -1, line: int = -1, column: int = -1) -> None:
         self.message = message
         assert level >= 0
         self.level = level or Error.ERROR

DHParser/parser.py

@@ -60,14 +60,14 @@ import abc
 import copy
 import os
 from functools import partial
+from typing import Any, Callable, cast, Dict, List, Set, Tuple, Union
 
-from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, load_if_file, \
-        re, typing
-from DHParser.stringview import StringView, EMPTY_STRING_VIEW
-from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, TOKEN_PTYPE, \
-    ZOMBIE_PARSER
 from DHParser.error import Error, is_error, has_errors, linebreaks, line_col
-from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union, Optional
+from DHParser.stringview import StringView, EMPTY_STRING_VIEW
+from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, \
+    TOKEN_PTYPE, ZOMBIE_PARSER
+from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, \
+    load_if_file, re
 
 __all__ = ('PreprocessorFunc',
            'HistoryRecord',
@@ -690,7 +690,6 @@ class Grammar:
                         parser.parser._name = entry
             cls.parser_initialization__ = "done"
 
-
     def __init__(self, root: Parser=None) -> None:
         # if not hasattr(self.__class__, 'parser_initialization__'):
         #     self.__class__.parser_initialization__ = "pending"
@@ -1603,7 +1602,7 @@ class Lookbehind(FlowOperator):
         while isinstance(p, Synonym):
             p = p.parser
         assert isinstance(p, RegExp), str(type(p))
-        self.regexp = p.main.regexp if isinstance(p, RE) else p.regexp
+        self.regexp = cast(RE, p).main.regexp if isinstance(p, RE) else p.regexp
         super(Lookbehind, self).__init__(parser, name)
 
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
@@ -1941,10 +1940,10 @@ class Compiler:
 
 
 def compile_source(source: str,
-                   preprocessor: PreprocessorFunc,   # str -> str
-                   parser: Grammar,                  # str -> Node (concrete syntax tree (CST))
+                   preprocessor: PreprocessorFunc,  # str -> str
+                   parser: Grammar,  # str -> Node (concrete syntax tree (CST))