Commit 7c0b34d6 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- cython stringview treatmend now settled on stringview.pxd solution.

parent 25cfadc1
# This module has been shamelessly stolen from the Cython Python
# to C compiler (https://cython.org) so that DHParser does not
# cause an import error on machines where Cython is not installed
# The copyrigh to Cython is held by Greg Ewing. It is licensed
# under the Apache 2.0 licnese. (see cython.org for mor information
# cython.* namespace for pure mode.
from __future__ import absolute_import
__version__ = "0.27"
try:
from __builtin__ import basestring
except ImportError:
basestring = str
# BEGIN shameless copy from Cython/minivect/minitypes.py
class _ArrayType(object):
is_array = True
subtypes = ['dtype']
def __init__(self, dtype, ndim, is_c_contig=False, is_f_contig=False,
inner_contig=False, broadcasting=None):
self.dtype = dtype
self.ndim = ndim
self.is_c_contig = is_c_contig
self.is_f_contig = is_f_contig
self.inner_contig = inner_contig or is_c_contig or is_f_contig
self.broadcasting = broadcasting
def __repr__(self):
axes = [":"] * self.ndim
if self.is_c_contig:
axes[-1] = "::1"
elif self.is_f_contig:
axes[0] = "::1"
return "%s[%s]" % (self.dtype, ", ".join(axes))
def index_type(base_type, item):
"""
Support array type creation by slicing, e.g. double[:, :] specifies
a 2D strided array of doubles. The syntax is the same as for
Cython memoryviews.
"""
class InvalidTypeSpecification(Exception):
pass
def verify_slice(s):
if s.start or s.stop or s.step not in (None, 1):
raise InvalidTypeSpecification(
"Only a step of 1 may be provided to indicate C or "
"Fortran contiguity")
if isinstance(item, tuple):
step_idx = None
for idx, s in enumerate(item):
verify_slice(s)
if s.step and (step_idx or idx not in (0, len(item) - 1)):
raise InvalidTypeSpecification(
"Step may only be provided once, and only in the "
"first or last dimension.")
if s.step == 1:
step_idx = idx
return _ArrayType(base_type, len(item),
is_c_contig=step_idx == len(item) - 1,
is_f_contig=step_idx == 0)
elif isinstance(item, slice):
verify_slice(item)
return _ArrayType(base_type, 1, is_c_contig=bool(item.step))
else:
# int[8] etc.
assert int(item) == item # array size must be a plain integer
array(base_type, item)
# END shameless copy
compiled = False
_Unspecified = object()
# Function decorators
def _empty_decorator(x):
return x
def locals(**arg_types):
return _empty_decorator
def test_assert_path_exists(*paths):
return _empty_decorator
def test_fail_if_path_exists(*paths):
return _empty_decorator
class _EmptyDecoratorAndManager(object):
def __call__(self, x):
return x
def __enter__(self):
pass
def __exit__(self, exc_type, exc_value, traceback):
pass
class _Optimization(object):
pass
cclass = ccall = cfunc = _EmptyDecoratorAndManager()
returns = wraparound = boundscheck = initializedcheck = nonecheck = \
overflowcheck = embedsignature = cdivision = cdivision_warnings = \
always_allows_keywords = profile = linetrace = infer_type = \
unraisable_tracebacks = freelist = \
lambda _: _EmptyDecoratorAndManager()
exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager()
optimization = _Optimization()
overflowcheck.fold = optimization.use_switch = \
optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager()
final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator
_cython_inline = None
def inline(f, *args, **kwds):
if isinstance(f, basestring):
global _cython_inline
if _cython_inline is None:
from Cython.Build.Inline import cython_inline as _cython_inline
return _cython_inline(f, *args, **kwds)
else:
assert len(args) == len(kwds) == 0
return f
def compile(f):
from Cython.Build.Inline import RuntimeCompiledFunction
return RuntimeCompiledFunction(f)
# Special functions
def cdiv(a, b):
q = a / b
if q < 0:
q += 1
return q
def cmod(a, b):
r = a % b
if (a * b) < 0:
r -= b
return r
# Emulated language constructs
def cast(type, *args, **kwargs):
kwargs.pop('typecheck', None)
assert not kwargs
if hasattr(type, '__call__'):
return type(*args)
else:
return args[0]
def sizeof(arg):
return 1
def typeof(arg):
return arg.__class__.__name__
# return type(arg)
def address(arg):
return pointer(type(arg))([arg])
def declare(type=None, value=_Unspecified, **kwds):
if type not in (None, object) and hasattr(type, '__call__'):
if value is not _Unspecified:
return type(value)
else:
return type()
else:
return value
class _nogil(object):
"""Support for 'with nogil' statement
"""
def __enter__(self):
pass
def __exit__(self, exc_class, exc, tb):
return exc_class is None
nogil = _nogil()
gil = _nogil()
del _nogil
# Emulated types
class CythonMetaType(type):
def __getitem__(type, ix):
return array(type, ix)
CythonTypeObject = CythonMetaType('CythonTypeObject', (object,), {})
class CythonType(CythonTypeObject):
def _pointer(self, n=1):
for i in range(n):
self = pointer(self)
return self
class PointerType(CythonType):
def __init__(self, value=None):
if isinstance(value, (ArrayType, PointerType)):
self._items = [cast(self._basetype, a) for a in value._items]
elif isinstance(value, list):
self._items = [cast(self._basetype, a) for a in value]
elif value is None or value == 0:
self._items = []
else:
raise ValueError
def __getitem__(self, ix):
if ix < 0:
raise IndexError("negative indexing not allowed in C")
return self._items[ix]
def __setitem__(self, ix, value):
if ix < 0:
raise IndexError("negative indexing not allowed in C")
self._items[ix] = cast(self._basetype, value)
def __eq__(self, value):
if value is None and not self._items:
return True
elif type(self) != type(value):
return False
else:
return not self._items and not value._items
def __repr__(self):
return "%s *" % (self._basetype,)
class ArrayType(PointerType):
def __init__(self):
self._items = [None] * self._n
class StructType(CythonType):
def __init__(self, cast_from=_Unspecified, **data):
if cast_from is not _Unspecified:
# do cast
if len(data) > 0:
raise ValueError('Cannot accept keyword arguments when casting.')
if type(cast_from) is not type(self):
raise ValueError('Cannot cast from %s' % cast_from)
for key, value in cast_from.__dict__.items():
setattr(self, key, value)
else:
for key, value in data.items():
setattr(self, key, value)
def __setattr__(self, key, value):
if key in self._members:
self.__dict__[key] = cast(self._members[key], value)
else:
raise AttributeError("Struct has no member '%s'" % key)
class UnionType(CythonType):
def __init__(self, cast_from=_Unspecified, **data):
if cast_from is not _Unspecified:
# do type cast
if len(data) > 0:
raise ValueError('Cannot accept keyword arguments when casting.')
if isinstance(cast_from, dict):
datadict = cast_from
elif type(cast_from) is type(self):
datadict = cast_from.__dict__
else:
raise ValueError('Cannot cast from %s' % cast_from)
else:
datadict = data
if len(datadict) > 1:
raise AttributeError("Union can only store one field at a time.")
for key, value in datadict.items():
setattr(self, key, value)
def __setattr__(self, key, value):
if key in '__dict__':
CythonType.__setattr__(self, key, value)
elif key in self._members:
self.__dict__ = {key: cast(self._members[key], value)}
else:
raise AttributeError("Union has no member '%s'" % key)
def pointer(basetype):
class PointerInstance(PointerType):
_basetype = basetype
return PointerInstance
def array(basetype, n):
class ArrayInstance(ArrayType):
_basetype = basetype
_n = n
return ArrayInstance
def struct(**members):
class StructInstance(StructType):
_members = members
for key in members:
setattr(StructInstance, key, None)
return StructInstance
def union(**members):
class UnionInstance(UnionType):
_members = members
for key in members:
setattr(UnionInstance, key, None)
return UnionInstance
class typedef(CythonType):
def __init__(self, type, name=None):
self._basetype = type
self.name = name
def __call__(self, *arg):
value = cast(self._basetype, *arg)
return value
def __repr__(self):
return self.name or str(self._basetype)
__getitem__ = index_type
class _FusedType(CythonType):
pass
def fused_type(*args):
if not args:
raise TypeError("Expected at least one type as argument")
# Find the numeric type with biggest rank if all types are numeric
rank = -1
for type in args:
if type not in (py_int, py_long, py_float, py_complex):
break
if type_ordering.index(type) > rank:
result_type = type
else:
return result_type
# Not a simple numeric type, return a fused type instance. The result
# isn't really meant to be used, as we can't keep track of the context in
# pure-mode. Casting won't do anything in this case.
return _FusedType()
def _specialized_from_args(signatures, args, kwargs):
"Perhaps this should be implemented in a TreeFragment in Cython code"
raise Exception("yet to be implemented")
py_int = typedef(int, "int")
try:
py_long = typedef(long, "long")
except NameError: # Py3
py_long = typedef(int, "long")
py_float = typedef(float, "float")
py_complex = typedef(complex, "double complex")
# Predefined types
int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t',
'size_t']
float_types = ['longdouble', 'double', 'float']
complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex']
other_types = ['bint', 'void']
to_repr = {
'longlong': 'long long',
'longdouble': 'long double',
'longdoublecomplex': 'long double complex',
'doublecomplex': 'double complex',
'floatcomplex': 'float complex',
}.get
gs = globals()
# note: cannot simply name the unicode type here as 2to3 gets in the way and replaces it by str
try:
import __builtin__ as builtins
except ImportError: # Py3
import builtins
gs['unicode'] = typedef(getattr(builtins, 'unicode', str), 'unicode')
del builtins
for name in int_types:
reprname = to_repr(name, name)
gs[name] = typedef(py_int, reprname)
if name not in ('Py_UNICODE', 'Py_UCS4') and not name.endswith('size_t'):
gs['u' + name] = typedef(py_int, "unsigned " + reprname)
gs['s' + name] = typedef(py_int, "signed " + reprname)
for name in float_types:
gs[name] = typedef(py_float, to_repr(name, name))
for name in complex_types:
gs[name] = typedef(py_complex, to_repr(name, name))
bint = typedef(bool, "bint")
void = typedef(int, "void")
for t in int_types + float_types + complex_types + other_types:
for i in range(1, 4):
gs["%s_%s" % ('p' * i, t)] = gs[t]._pointer(i)
void = typedef(None, "void")
NULL = gs['p_void'](0)
# looks like 'gs' has some users out there by now...
# del gs
integral = floating = numeric = _FusedType()
type_ordering = [py_int, py_long, py_float, py_complex]
class CythonDotParallel(object):
"""
The cython.parallel module.
"""
__all__ = ['parallel', 'prange', 'threadid']
def parallel(self, num_threads=None):
return nogil
def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False):
if stop is None:
stop = start
start = 0
return range(start, stop, step)
def threadid(self):
return 0
# def threadsavailable(self):
# return 1
import sys
sys.modules['cython.parallel'] = CythonDotParallel()
del sys
......@@ -20,17 +20,16 @@ compilation of domain specific languages based on an EBNF-grammar.
"""
import os
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
re, typing
from DHParser.error import Error, is_error, has_errors, only_errors
from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, TransformationFunc
from DHParser.error import Error, is_error, has_errors, only_errors
from typing import Any, cast, Tuple, Union, Iterator, Iterable
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
re
__all__ = ('GrammarError',
'CompilationError',
......@@ -170,7 +169,7 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
# read grammar
grammar_src = load_if_file(grammar_representation)
if is_python_code(grammar_src):
parser_py, messages, AST = grammar_src, [], None
parser_py, messages, AST = grammar_src, [], None # type: str, List[Error], Node
else:
with logging(False):
parser_py, messages, AST = compile_source(grammar_src, None,
......@@ -371,7 +370,7 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
"""
Compiles the a source file with a given compiler and writes the
result to a file.
......@@ -416,7 +415,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
cfactory = get_ebnf_compiler
compiler1 = cfactory()
compiler1.set_grammar_name(compiler_name, source_file)
result, messages, ast = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
if has_errors(messages):
return messages
......@@ -522,7 +521,7 @@ def recompile_grammar(ebnf_filename, force=False) -> bool:
base, ext = os.path.splitext(ebnf_filename)
compiler_name = base + 'Compiler.py'
error_file_name = base + '_ebnf_ERRORS.txt'
messages = [] # type: Iterable[str]
messages = [] # type: Iterable[Error]
if (not os.path.exists(compiler_name) or force or
grammar_changed(compiler_name, ebnf_filename)):
# print("recompiling parser for: " + ebnf_filename)
......
......@@ -44,7 +44,7 @@ class Error:
MANDATORY_CONTINUATION = 1001
def __init__(self, message: str, level: int = ERROR, code: Hashable = 0,
pos: int = -1, line: int = -1, column: int = -1):
pos: int = -1, line: int = -1, column: int = -1) -> None:
self.message = message
assert level >= 0
self.level = level or Error.ERROR
......
......@@ -60,14 +60,14 @@ import abc
import copy
import os
from functools import partial
from typing import Any, Callable, cast, Dict, List, Set, Tuple, Union
from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, load_if_file, \
re, typing
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, TOKEN_PTYPE, \
ZOMBIE_PARSER
from DHParser.error import Error, is_error, has_errors, linebreaks, line_col
from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union, Optional
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_PARSER
from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, \
load_if_file, re
__all__ = ('PreprocessorFunc',
'HistoryRecord',
......@@ -690,7 +690,6 @@ class Grammar:
parser.parser._name = entry
cls.parser_initialization__ = "done"
def __init__(self, root: Parser=None) -> None:
# if not hasattr(self.__class__, 'parser_initialization__'):
# self.__class__.parser_initialization__ = "pending"
......@@ -1603,7 +1602,7 @@ class Lookbehind(FlowOperator):
while isinstance(p, Synonym):
p = p.parser
assert isinstance(p, RegExp), str(type(p))
self.regexp = p.main.regexp if isinstance(p, RE) else p.regexp
self.regexp = cast(RE, p).main.regexp if isinstance(p, RE) else p.regexp
super(Lookbehind, self).__init__(parser, name)
def __call__(self, text: StringView) -> Tuple[Node, StringView]:
......@@ -1941,10 +1940,10 @@ class Compiler:
def compile_source(source: str,