Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
7c0b34d6
Commit
7c0b34d6
authored
Sep 29, 2017
by
Eckhart Arnold
Browse files
- cython stringview treatmend now settled on stringview.pxd solution.
parent
25cfadc1
Changes
9
Hide whitespace changes
Inline
Side-by-side
DHParser/VERALTET/foreign_cython.py
0 → 100644
View file @
7c0b34d6
# This module has been shamelessly stolen from the Cython Python
# to C compiler (https://cython.org) so that DHParser does not
# cause an import error on machines where Cython is not installed
# The copyrigh to Cython is held by Greg Ewing. It is licensed
# under the Apache 2.0 licnese. (see cython.org for mor information
# cython.* namespace for pure mode.
from
__future__
import
absolute_import
__version__
=
"0.27"
try
:
from
__builtin__
import
basestring
except
ImportError
:
basestring
=
str
# BEGIN shameless copy from Cython/minivect/minitypes.py
class
_ArrayType
(
object
):
is_array
=
True
subtypes
=
[
'dtype'
]
def
__init__
(
self
,
dtype
,
ndim
,
is_c_contig
=
False
,
is_f_contig
=
False
,
inner_contig
=
False
,
broadcasting
=
None
):
self
.
dtype
=
dtype
self
.
ndim
=
ndim
self
.
is_c_contig
=
is_c_contig
self
.
is_f_contig
=
is_f_contig
self
.
inner_contig
=
inner_contig
or
is_c_contig
or
is_f_contig
self
.
broadcasting
=
broadcasting
def
__repr__
(
self
):
axes
=
[
":"
]
*
self
.
ndim
if
self
.
is_c_contig
:
axes
[
-
1
]
=
"::1"
elif
self
.
is_f_contig
:
axes
[
0
]
=
"::1"
return
"%s[%s]"
%
(
self
.
dtype
,
", "
.
join
(
axes
))
def
index_type
(
base_type
,
item
):
"""
Support array type creation by slicing, e.g. double[:, :] specifies
a 2D strided array of doubles. The syntax is the same as for
Cython memoryviews.
"""
class
InvalidTypeSpecification
(
Exception
):
pass
def
verify_slice
(
s
):
if
s
.
start
or
s
.
stop
or
s
.
step
not
in
(
None
,
1
):
raise
InvalidTypeSpecification
(
"Only a step of 1 may be provided to indicate C or "
"Fortran contiguity"
)
if
isinstance
(
item
,
tuple
):
step_idx
=
None
for
idx
,
s
in
enumerate
(
item
):
verify_slice
(
s
)
if
s
.
step
and
(
step_idx
or
idx
not
in
(
0
,
len
(
item
)
-
1
)):
raise
InvalidTypeSpecification
(
"Step may only be provided once, and only in the "
"first or last dimension."
)
if
s
.
step
==
1
:
step_idx
=
idx
return
_ArrayType
(
base_type
,
len
(
item
),
is_c_contig
=
step_idx
==
len
(
item
)
-
1
,
is_f_contig
=
step_idx
==
0
)
elif
isinstance
(
item
,
slice
):
verify_slice
(
item
)
return
_ArrayType
(
base_type
,
1
,
is_c_contig
=
bool
(
item
.
step
))
else
:
# int[8] etc.
assert
int
(
item
)
==
item
# array size must be a plain integer
array
(
base_type
,
item
)
# END shameless copy
compiled
=
False
_Unspecified
=
object
()
# Function decorators
def
_empty_decorator
(
x
):
return
x
def
locals
(
**
arg_types
):
return
_empty_decorator
def
test_assert_path_exists
(
*
paths
):
return
_empty_decorator
def
test_fail_if_path_exists
(
*
paths
):
return
_empty_decorator
class
_EmptyDecoratorAndManager
(
object
):
def
__call__
(
self
,
x
):
return
x
def
__enter__
(
self
):
pass
def
__exit__
(
self
,
exc_type
,
exc_value
,
traceback
):
pass
class
_Optimization
(
object
):
pass
cclass
=
ccall
=
cfunc
=
_EmptyDecoratorAndManager
()
returns
=
wraparound
=
boundscheck
=
initializedcheck
=
nonecheck
=
\
overflowcheck
=
embedsignature
=
cdivision
=
cdivision_warnings
=
\
always_allows_keywords
=
profile
=
linetrace
=
infer_type
=
\
unraisable_tracebacks
=
freelist
=
\
lambda
_
:
_EmptyDecoratorAndManager
()
exceptval
=
lambda
_
=
None
,
check
=
True
:
_EmptyDecoratorAndManager
()
optimization
=
_Optimization
()
overflowcheck
.
fold
=
optimization
.
use_switch
=
\
optimization
.
unpack_method_calls
=
lambda
arg
:
_EmptyDecoratorAndManager
()
final
=
internal
=
type_version_tag
=
no_gc_clear
=
no_gc
=
_empty_decorator
_cython_inline
=
None
def
inline
(
f
,
*
args
,
**
kwds
):
if
isinstance
(
f
,
basestring
):
global
_cython_inline
if
_cython_inline
is
None
:
from
Cython.Build.Inline
import
cython_inline
as
_cython_inline
return
_cython_inline
(
f
,
*
args
,
**
kwds
)
else
:
assert
len
(
args
)
==
len
(
kwds
)
==
0
return
f
def
compile
(
f
):
from
Cython.Build.Inline
import
RuntimeCompiledFunction
return
RuntimeCompiledFunction
(
f
)
# Special functions
def
cdiv
(
a
,
b
):
q
=
a
/
b
if
q
<
0
:
q
+=
1
return
q
def
cmod
(
a
,
b
):
r
=
a
%
b
if
(
a
*
b
)
<
0
:
r
-=
b
return
r
# Emulated language constructs
def
cast
(
type
,
*
args
,
**
kwargs
):
kwargs
.
pop
(
'typecheck'
,
None
)
assert
not
kwargs
if
hasattr
(
type
,
'__call__'
):
return
type
(
*
args
)
else
:
return
args
[
0
]
def
sizeof
(
arg
):
return
1
def
typeof
(
arg
):
return
arg
.
__class__
.
__name__
# return type(arg)
def
address
(
arg
):
return
pointer
(
type
(
arg
))([
arg
])
def
declare
(
type
=
None
,
value
=
_Unspecified
,
**
kwds
):
if
type
not
in
(
None
,
object
)
and
hasattr
(
type
,
'__call__'
):
if
value
is
not
_Unspecified
:
return
type
(
value
)
else
:
return
type
()
else
:
return
value
class
_nogil
(
object
):
"""Support for 'with nogil' statement
"""
def
__enter__
(
self
):
pass
def
__exit__
(
self
,
exc_class
,
exc
,
tb
):
return
exc_class
is
None
nogil
=
_nogil
()
gil
=
_nogil
()
del
_nogil
# Emulated types
class
CythonMetaType
(
type
):
def
__getitem__
(
type
,
ix
):
return
array
(
type
,
ix
)
CythonTypeObject
=
CythonMetaType
(
'CythonTypeObject'
,
(
object
,),
{})
class
CythonType
(
CythonTypeObject
):
def
_pointer
(
self
,
n
=
1
):
for
i
in
range
(
n
):
self
=
pointer
(
self
)
return
self
class
PointerType
(
CythonType
):
def
__init__
(
self
,
value
=
None
):
if
isinstance
(
value
,
(
ArrayType
,
PointerType
)):
self
.
_items
=
[
cast
(
self
.
_basetype
,
a
)
for
a
in
value
.
_items
]
elif
isinstance
(
value
,
list
):
self
.
_items
=
[
cast
(
self
.
_basetype
,
a
)
for
a
in
value
]
elif
value
is
None
or
value
==
0
:
self
.
_items
=
[]
else
:
raise
ValueError
def
__getitem__
(
self
,
ix
):
if
ix
<
0
:
raise
IndexError
(
"negative indexing not allowed in C"
)
return
self
.
_items
[
ix
]
def
__setitem__
(
self
,
ix
,
value
):
if
ix
<
0
:
raise
IndexError
(
"negative indexing not allowed in C"
)
self
.
_items
[
ix
]
=
cast
(
self
.
_basetype
,
value
)
def
__eq__
(
self
,
value
):
if
value
is
None
and
not
self
.
_items
:
return
True
elif
type
(
self
)
!=
type
(
value
):
return
False
else
:
return
not
self
.
_items
and
not
value
.
_items
def
__repr__
(
self
):
return
"%s *"
%
(
self
.
_basetype
,)
class
ArrayType
(
PointerType
):
def
__init__
(
self
):
self
.
_items
=
[
None
]
*
self
.
_n
class
StructType
(
CythonType
):
def
__init__
(
self
,
cast_from
=
_Unspecified
,
**
data
):
if
cast_from
is
not
_Unspecified
:
# do cast
if
len
(
data
)
>
0
:
raise
ValueError
(
'Cannot accept keyword arguments when casting.'
)
if
type
(
cast_from
)
is
not
type
(
self
):
raise
ValueError
(
'Cannot cast from %s'
%
cast_from
)
for
key
,
value
in
cast_from
.
__dict__
.
items
():
setattr
(
self
,
key
,
value
)
else
:
for
key
,
value
in
data
.
items
():
setattr
(
self
,
key
,
value
)
def
__setattr__
(
self
,
key
,
value
):
if
key
in
self
.
_members
:
self
.
__dict__
[
key
]
=
cast
(
self
.
_members
[
key
],
value
)
else
:
raise
AttributeError
(
"Struct has no member '%s'"
%
key
)
class
UnionType
(
CythonType
):
def
__init__
(
self
,
cast_from
=
_Unspecified
,
**
data
):
if
cast_from
is
not
_Unspecified
:
# do type cast
if
len
(
data
)
>
0
:
raise
ValueError
(
'Cannot accept keyword arguments when casting.'
)
if
isinstance
(
cast_from
,
dict
):
datadict
=
cast_from
elif
type
(
cast_from
)
is
type
(
self
):
datadict
=
cast_from
.
__dict__
else
:
raise
ValueError
(
'Cannot cast from %s'
%
cast_from
)
else
:
datadict
=
data
if
len
(
datadict
)
>
1
:
raise
AttributeError
(
"Union can only store one field at a time."
)
for
key
,
value
in
datadict
.
items
():
setattr
(
self
,
key
,
value
)
def
__setattr__
(
self
,
key
,
value
):
if
key
in
'__dict__'
:
CythonType
.
__setattr__
(
self
,
key
,
value
)
elif
key
in
self
.
_members
:
self
.
__dict__
=
{
key
:
cast
(
self
.
_members
[
key
],
value
)}
else
:
raise
AttributeError
(
"Union has no member '%s'"
%
key
)
def
pointer
(
basetype
):
class
PointerInstance
(
PointerType
):
_basetype
=
basetype
return
PointerInstance
def
array
(
basetype
,
n
):
class
ArrayInstance
(
ArrayType
):
_basetype
=
basetype
_n
=
n
return
ArrayInstance
def
struct
(
**
members
):
class
StructInstance
(
StructType
):
_members
=
members
for
key
in
members
:
setattr
(
StructInstance
,
key
,
None
)
return
StructInstance
def
union
(
**
members
):
class
UnionInstance
(
UnionType
):
_members
=
members
for
key
in
members
:
setattr
(
UnionInstance
,
key
,
None
)
return
UnionInstance
class
typedef
(
CythonType
):
def
__init__
(
self
,
type
,
name
=
None
):
self
.
_basetype
=
type
self
.
name
=
name
def
__call__
(
self
,
*
arg
):
value
=
cast
(
self
.
_basetype
,
*
arg
)
return
value
def
__repr__
(
self
):
return
self
.
name
or
str
(
self
.
_basetype
)
__getitem__
=
index_type
class
_FusedType
(
CythonType
):
pass
def
fused_type
(
*
args
):
if
not
args
:
raise
TypeError
(
"Expected at least one type as argument"
)
# Find the numeric type with biggest rank if all types are numeric
rank
=
-
1
for
type
in
args
:
if
type
not
in
(
py_int
,
py_long
,
py_float
,
py_complex
):
break
if
type_ordering
.
index
(
type
)
>
rank
:
result_type
=
type
else
:
return
result_type
# Not a simple numeric type, return a fused type instance. The result
# isn't really meant to be used, as we can't keep track of the context in
# pure-mode. Casting won't do anything in this case.
return
_FusedType
()
def
_specialized_from_args
(
signatures
,
args
,
kwargs
):
"Perhaps this should be implemented in a TreeFragment in Cython code"
raise
Exception
(
"yet to be implemented"
)
py_int
=
typedef
(
int
,
"int"
)
try
:
py_long
=
typedef
(
long
,
"long"
)
except
NameError
:
# Py3
py_long
=
typedef
(
int
,
"long"
)
py_float
=
typedef
(
float
,
"float"
)
py_complex
=
typedef
(
complex
,
"double complex"
)
# Predefined types
int_types
=
[
'char'
,
'short'
,
'Py_UNICODE'
,
'int'
,
'Py_UCS4'
,
'long'
,
'longlong'
,
'Py_ssize_t'
,
'size_t'
]
float_types
=
[
'longdouble'
,
'double'
,
'float'
]
complex_types
=
[
'longdoublecomplex'
,
'doublecomplex'
,
'floatcomplex'
,
'complex'
]
other_types
=
[
'bint'
,
'void'
]
to_repr
=
{
'longlong'
:
'long long'
,
'longdouble'
:
'long double'
,
'longdoublecomplex'
:
'long double complex'
,
'doublecomplex'
:
'double complex'
,
'floatcomplex'
:
'float complex'
,
}.
get
gs
=
globals
()
# note: cannot simply name the unicode type here as 2to3 gets in the way and replaces it by str
try
:
import
__builtin__
as
builtins
except
ImportError
:
# Py3
import
builtins
gs
[
'unicode'
]
=
typedef
(
getattr
(
builtins
,
'unicode'
,
str
),
'unicode'
)
del
builtins
for
name
in
int_types
:
reprname
=
to_repr
(
name
,
name
)
gs
[
name
]
=
typedef
(
py_int
,
reprname
)
if
name
not
in
(
'Py_UNICODE'
,
'Py_UCS4'
)
and
not
name
.
endswith
(
'size_t'
):
gs
[
'u'
+
name
]
=
typedef
(
py_int
,
"unsigned "
+
reprname
)
gs
[
's'
+
name
]
=
typedef
(
py_int
,
"signed "
+
reprname
)
for
name
in
float_types
:
gs
[
name
]
=
typedef
(
py_float
,
to_repr
(
name
,
name
))
for
name
in
complex_types
:
gs
[
name
]
=
typedef
(
py_complex
,
to_repr
(
name
,
name
))
bint
=
typedef
(
bool
,
"bint"
)
void
=
typedef
(
int
,
"void"
)
for
t
in
int_types
+
float_types
+
complex_types
+
other_types
:
for
i
in
range
(
1
,
4
):
gs
[
"%s_%s"
%
(
'p'
*
i
,
t
)]
=
gs
[
t
].
_pointer
(
i
)
void
=
typedef
(
None
,
"void"
)
NULL
=
gs
[
'p_void'
](
0
)
# looks like 'gs' has some users out there by now...
# del gs
integral
=
floating
=
numeric
=
_FusedType
()
type_ordering
=
[
py_int
,
py_long
,
py_float
,
py_complex
]
class
CythonDotParallel
(
object
):
"""
The cython.parallel module.
"""
__all__
=
[
'parallel'
,
'prange'
,
'threadid'
]
def
parallel
(
self
,
num_threads
=
None
):
return
nogil
def
prange
(
self
,
start
=
0
,
stop
=
None
,
step
=
1
,
schedule
=
None
,
nogil
=
False
):
if
stop
is
None
:
stop
=
start
start
=
0
return
range
(
start
,
stop
,
step
)
def
threadid
(
self
):
return
0
# def threadsavailable(self):
# return 1
import
sys
sys
.
modules
[
'cython.parallel'
]
=
CythonDotParallel
()
del
sys
DHParser/dsl.py
View file @
7c0b34d6
...
...
@@ -20,17 +20,16 @@ compilation of domain specific languages based on an EBNF-grammar.
"""
import
os
from
typing
import
Any
,
cast
,
List
,
Tuple
,
Union
,
Iterator
,
Iterable
from
DHParser.ebnf
import
EBNFCompiler
,
grammar_changed
,
\
get_ebnf_preprocessor
,
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
,
\
PreprocessorFactoryFunc
,
ParserFactoryFunc
,
TransformerFactoryFunc
,
CompilerFactoryFunc
from
DHParser.toolkit
import
logging
,
load_if_file
,
is_python_code
,
compile_python_object
,
\
re
,
typing
from
DHParser.error
import
Error
,
is_error
,
has_errors
,
only_errors
from
DHParser.parser
import
Grammar
,
Compiler
,
compile_source
,
nil_preprocessor
,
PreprocessorFunc
from
DHParser.syntaxtree
import
Node
,
TransformationFunc
from
DHParser.error
import
Error
,
is_error
,
has_errors
,
only_errors
from
typing
import
Any
,
cast
,
Tuple
,
Union
,
Iterator
,
Iterable
from
DHParser.toolkit
import
logging
,
load_if_file
,
is_python_code
,
compile_python_object
,
\
re
__all__
=
(
'GrammarError'
,
'CompilationError'
,
...
...
@@ -170,7 +169,7 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
# read grammar
grammar_src
=
load_if_file
(
grammar_representation
)
if
is_python_code
(
grammar_src
):
parser_py
,
messages
,
AST
=
grammar_src
,
[],
None
parser_py
,
messages
,
AST
=
grammar_src
,
[],
None
# type: str, List[Error], Node
else
:
with
logging
(
False
):
parser_py
,
messages
,
AST
=
compile_source
(
grammar_src
,
None
,
...
...
@@ -371,7 +370,7 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
return
compileDSL
(
text_or_file
,
preprocessor
(),
parser
(),
ast
(),
compiler
())
def
compile_on_disk
(
source_file
:
str
,
compiler_suite
=
""
,
extension
=
".xml"
):
def
compile_on_disk
(
source_file
:
str
,
compiler_suite
=
""
,
extension
=
".xml"
)
->
Iterable
[
Error
]
:
"""
Compiles the a source file with a given compiler and writes the
result to a file.
...
...
@@ -416,7 +415,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
cfactory
=
get_ebnf_compiler
compiler1
=
cfactory
()
compiler1
.
set_grammar_name
(
compiler_name
,
source_file
)
result
,
messages
,
ast
=
compile_source
(
source
,
sfactory
(),
pfactory
(),
tfactory
(),
compiler1
)
result
,
messages
,
AST
=
compile_source
(
source
,
sfactory
(),
pfactory
(),
tfactory
(),
compiler1
)
if
has_errors
(
messages
):
return
messages
...
...
@@ -522,7 +521,7 @@ def recompile_grammar(ebnf_filename, force=False) -> bool:
base
,
ext
=
os
.
path
.
splitext
(
ebnf_filename
)
compiler_name
=
base
+
'Compiler.py'
error_file_name
=
base
+
'_ebnf_ERRORS.txt'
messages
=
[]
# type: Iterable[
st
r]
messages
=
[]
# type: Iterable[
Erro
r]
if
(
not
os
.
path
.
exists
(
compiler_name
)
or
force
or
grammar_changed
(
compiler_name
,
ebnf_filename
)):
# print("recompiling parser for: " + ebnf_filename)
...
...
DHParser/error.py
View file @
7c0b34d6
...
...
@@ -44,7 +44,7 @@ class Error:
MANDATORY_CONTINUATION
=
1001
def
__init__
(
self
,
message
:
str
,
level
:
int
=
ERROR
,
code
:
Hashable
=
0
,
pos
:
int
=
-
1
,
line
:
int
=
-
1
,
column
:
int
=
-
1
):
pos
:
int
=
-
1
,
line
:
int
=
-
1
,
column
:
int
=
-
1
)
->
None
:
self
.
message
=
message
assert
level
>=
0
self
.
level
=
level
or
Error
.
ERROR
...
...
DHParser/parser.py
View file @
7c0b34d6
...
...
@@ -60,14 +60,14 @@ import abc
import
copy
import
os
from
functools
import
partial
from
typing
import
Any
,
Callable
,
cast
,
Dict
,
List
,
Set
,
Tuple
,
Union
from
DHParser.toolkit
import
is_logging
,
log_dir
,
logfile_basename
,
escape_re
,
sane_parser_name
,
load_if_file
,
\
re
,
typing
from
DHParser.stringview
import
StringView
,
EMPTY_STRING_VIEW
from
DHParser.syntaxtree
import
Node
,
TransformationFunc
,
ParserBase
,
WHITESPACE_PTYPE
,
TOKEN_PTYPE
,
\
ZOMBIE_PARSER
from
DHParser.error
import
Error
,
is_error
,
has_errors
,
linebreaks
,
line_col
from
typing
import
Any
,
Callable
,
cast
,
Dict
,
Iterator
,
List
,
Set
,
Tuple
,
Union
,
Optional
from
DHParser.stringview
import
StringView
,
EMPTY_STRING_VIEW
from
DHParser.syntaxtree
import
Node
,
TransformationFunc
,
ParserBase
,
WHITESPACE_PTYPE
,
\
TOKEN_PTYPE
,
ZOMBIE_PARSER
from
DHParser.toolkit
import
is_logging
,
log_dir
,
logfile_basename
,
escape_re
,
sane_parser_name
,
\
load_if_file
,
re
__all__
=
(
'PreprocessorFunc'
,
'HistoryRecord'
,
...
...
@@ -690,7 +690,6 @@ class Grammar:
parser
.
parser
.
_name
=
entry
cls
.
parser_initialization__
=
"done"
def
__init__
(
self
,
root
:
Parser
=
None
)
->
None
:
# if not hasattr(self.__class__, 'parser_initialization__'):
# self.__class__.parser_initialization__ = "pending"
...
...
@@ -1603,7 +1602,7 @@ class Lookbehind(FlowOperator):
while
isinstance
(
p
,
Synonym
):
p
=
p
.
parser
assert
isinstance
(
p
,
RegExp
),
str
(
type
(
p
))
self
.
regexp
=
p
.
main
.
regexp
if
isinstance
(
p
,
RE
)
else
p
.
regexp
self
.
regexp
=
cast
(
RE
,
p
)
.
main
.
regexp
if
isinstance
(
p
,
RE
)
else
p
.
regexp
super
(
Lookbehind
,
self
).
__init__
(
parser
,
name
)
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Node
,
StringView
]:
...
...
@@ -1941,10 +1940,10 @@ class Compiler:
def
compile_source
(
source
:
str
,
preprocessor
:
PreprocessorFunc
,
# str -> str
parser
:
Grammar
,
# str -> Node (concrete syntax tree (CST))