Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
da2df627
Commit
da2df627
authored
Jul 16, 2017
by
Eckhart Arnold
Browse files
- script for creating single-module ("stand-alone") version of DHParser
parent
98411756
Changes
17
Hide whitespace changes
Inline
Side-by-side
DHParser/__init__.py
View file @
da2df627
...
...
@@ -24,6 +24,7 @@ from .parser import *
from
.syntaxtree
import
*
# Flat namespace for the DHParser Package. Is this a good idea...?
from
.toolkit
import
*
from
.transform
import
*
# from .testing import *
from
.versionnumber
import
__version__
...
...
DHParser/dsl.py
View file @
da2df627
...
...
@@ -20,6 +20,7 @@ compilation of domain specific languages based on an EBNF-grammar.
"""
import
os
try
:
import
regex
as
re
except
ImportError
:
...
...
@@ -36,15 +37,14 @@ from DHParser.toolkit import logging, load_if_file, is_python_code, compile_pyth
from
DHParser.parser
import
Grammar
,
Compiler
,
compile_source
,
nil_preprocessor
,
PreprocessorFunc
from
DHParser.syntaxtree
import
Node
,
TransformationFunc
__all__
=
[
'GrammarError'
,
__all__
=
(
'GrammarError'
,
'CompilationError'
,
'load_compiler_suite'
,
'compileDSL'
,
'raw_compileEBNF'
,
'compileEBNF'
,
'parser_factory'
,
'compile_on_disk'
]
'compile_on_disk'
)
SECTION_MARKER
=
"""
\n
...
...
@@ -74,14 +74,14 @@ try:
import regex as re
except ImportError:
import re
from DHParser
.toolkit
import logging, is_filename, load_if_file
from DHParser.parser import
Grammar, Compiler, nil_preprocessor,
\\
from DHParser import logging, is_filename, load_if_file
,
\\
Grammar, Compiler, nil_preprocessor,
\\
Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym,
\\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture,
\\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source,
\\
last_value, counterpart, accumulate, PreprocessorFunc
from DHParser.syntaxtree import
Node, TransformationFunc
from DHParser.transform import
traverse, remove_children_if,
\\
last_value, counterpart, accumulate, PreprocessorFunc
,
\\
Node, TransformationFunc
,
\\
traverse, remove_children_if,
\\
reduce_single_child, replace_by_single_child, remove_whitespace,
\\
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace,
\\
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE,
\\
...
...
@@ -483,3 +483,45 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
if
f
:
f
.
close
()
return
[]
def
recompile_grammar
(
ebnf_filename
,
force
=
False
)
->
bool
:
"""
Recompiles an ebnf-grammar if necessary, that is, if either no
corresponding 'XXXXCompiler.py'-file exists or if that file is
outdated.
Parameters:
ebnf_filename(str): The filename of the ebnf-source of the
grammar. In case this is a directory and not a file, all
files within this directory ending with .ebnf will be
compiled.
force(bool): If False (default), the grammar will only be
recompiled if it has been changed.
"""
if
os
.
path
.
isdir
(
ebnf_filename
):
success
=
True
for
entry
in
os
.
listdir
(
ebnf_filename
):
if
entry
.
lower
().
endswith
(
'.ebnf'
)
and
os
.
path
.
isfile
(
entry
):
success
=
success
and
recompile_grammar
(
entry
,
force
)
return
success
base
,
ext
=
os
.
path
.
splitext
(
ebnf_filename
)
compiler_name
=
base
+
'Compiler.py'
error_file_name
=
base
+
'_ebnf_ERRORS.txt'
errors
=
[]
if
(
not
os
.
path
.
exists
(
compiler_name
)
or
force
or
grammar_changed
(
compiler_name
,
ebnf_filename
)):
# print("recompiling parser for: " + ebnf_filename)
errors
=
compile_on_disk
(
ebnf_filename
)
if
errors
:
# print("Errors while compiling: " + ebnf_filename + '!')
with
open
(
error_file_name
,
'w'
)
as
f
:
for
e
in
errors
:
f
.
write
(
e
)
f
.
write
(
'
\n
'
)
return
False
if
not
errors
and
os
.
path
.
exists
(
error_file_name
):
os
.
remove
(
error_file_name
)
return
True
DHParser/parser.py
View file @
da2df627
...
...
@@ -61,10 +61,10 @@ except ImportError:
import
re
try
:
from
typing
import
Any
,
Callable
,
cast
,
Dict
,
Iterator
,
List
,
Set
,
Tuple
,
Union
try
:
from
typing
import
Collection
except
ImportError
:
pass
#
try:
#
from typing import Collection
#
except ImportError:
#
pass
except
ImportError
:
from
.typing34
import
Any
,
Callable
,
cast
,
Dict
,
Iterator
,
List
,
Set
,
Tuple
,
Union
...
...
@@ -73,7 +73,7 @@ from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, Pa
Node
,
TransformationFunc
from
DHParser.toolkit
import
load_if_file
,
error_messages
__all__
=
[
'PreprocessorFunc'
,
__all__
=
(
'PreprocessorFunc'
,
'HistoryRecord'
,
'Parser'
,
'Grammar'
,
...
...
@@ -109,7 +109,7 @@ __all__ = ['PreprocessorFunc',
'Pop'
,
'Forward'
,
'Compiler'
,
'compile_source'
]
'compile_source'
)
...
...
DHParser/syntaxtree.py
View file @
da2df627
...
...
@@ -33,14 +33,14 @@ except ImportError:
from
DHParser.toolkit
import
log_dir
,
line_col
__all__
=
[
'WHITESPACE_PTYPE'
,
__all__
=
(
'WHITESPACE_PTYPE'
,
'TOKEN_PTYPE'
,
'ZOMBIE_PARSER'
,
'ParserBase'
,
'Error'
,
'Node'
,
'TransformationFunc'
]
'mock_syntax_tree'
,
'TransformationFunc'
)
class
ParserBase
:
...
...
@@ -461,4 +461,56 @@ class Node:
# return nav(path.split('/'))
def
mock_syntax_tree
(
sexpr
):
"""
Generates a tree of nodes from an S-expression.
Example:
>>> mock_syntax_tree("(a (b c))").as_sxpr()
'(a
\\
n (b
\\
n "c"
\\
n )
\\
n)'
"""
def
next_block
(
s
):
s
=
s
.
strip
()
while
s
[
0
]
!=
')'
:
if
s
[
0
]
!=
'('
:
raise
ValueError
(
'"(" expected, not '
+
s
[:
10
])
# assert s[0] == '(', s
level
=
1
i
=
1
while
level
>
0
:
if
s
[
i
]
==
'('
:
level
+=
1
elif
s
[
i
]
==
')'
:
level
-=
1
i
+=
1
yield
s
[:
i
]
s
=
s
[
i
:].
strip
()
sexpr
=
sexpr
.
strip
()
if
sexpr
[
0
]
!=
'('
:
raise
ValueError
(
'"(" expected, not '
+
sexpr
[:
10
])
# assert sexpr[0] == '(', sexpr
sexpr
=
sexpr
[
1
:].
strip
()
m
=
re
.
match
(
'[\w:]+'
,
sexpr
)
name
,
class_name
=
(
sexpr
[:
m
.
end
()].
split
(
':'
)
+
[
''
])[:
2
]
sexpr
=
sexpr
[
m
.
end
():].
strip
()
if
sexpr
[
0
]
==
'('
:
result
=
tuple
(
mock_syntax_tree
(
block
)
for
block
in
next_block
(
sexpr
))
else
:
lines
=
[]
while
sexpr
and
sexpr
[
0
]
!=
')'
:
for
qm
in
[
'"""'
,
"'''"
,
'"'
,
"'"
]:
m
=
re
.
match
(
qm
+
r
'.*?'
+
qm
,
sexpr
)
if
m
:
i
=
len
(
qm
)
lines
.
append
(
sexpr
[
i
:
m
.
end
()
-
i
])
sexpr
=
sexpr
[
m
.
end
():].
strip
()
break
else
:
m
=
re
.
match
(
r
'(?:(?!\)).)*'
,
sexpr
)
lines
.
append
(
sexpr
[:
m
.
end
()])
sexpr
=
sexpr
[
m
.
end
():]
result
=
"
\n
"
.
join
(
lines
)
return
Node
(
MockParser
(
name
,
':'
+
class_name
),
result
)
TransformationFunc
=
Union
[
Callable
[[
Node
],
Any
],
partial
]
DHParser/testing.py
View file @
da2df627
...
...
@@ -26,15 +26,11 @@ try:
except
ImportError
:
import
re
from
DHParser
import
Node
,
error_messages
from
DHParser
import
error_messages
from
DHParser.toolkit
import
compact_sexpr
,
is_logging
from
DHParser.syntaxtree
import
MockParser
from
DHParser.ebnf
import
grammar_changed
from
DHParser.dsl
import
compile_on_disk
from
DHParser.syntaxtree
import
mock_syntax_tree
__all__
=
(
'mock_syntax_tree'
,
'recompile_grammar'
,
'unit_from_configfile'
,
__all__
=
(
'unit_from_configfile'
,
'unit_from_json'
,
'unit_from_file'
,
'get_report'
,
...
...
@@ -42,100 +38,6 @@ __all__ = ('mock_syntax_tree',
'grammar_suite'
,
'runner'
)
def
mock_syntax_tree
(
sexpr
):
"""
Generates a tree of nodes from an S-expression.
Example:
>>> mock_syntax_tree("(a (b c))").as_sxpr()
'(a
\\
n (b
\\
n "c"
\\
n )
\\
n)'
"""
def
next_block
(
s
):
s
=
s
.
strip
()
while
s
[
0
]
!=
')'
:
if
s
[
0
]
!=
'('
:
raise
ValueError
(
'"(" expected, not '
+
s
[:
10
])
# assert s[0] == '(', s
level
=
1
i
=
1
while
level
>
0
:
if
s
[
i
]
==
'('
:
level
+=
1
elif
s
[
i
]
==
')'
:
level
-=
1
i
+=
1
yield
s
[:
i
]
s
=
s
[
i
:].
strip
()
sexpr
=
sexpr
.
strip
()
if
sexpr
[
0
]
!=
'('
:
raise
ValueError
(
'"(" expected, not '
+
sexpr
[:
10
])
# assert sexpr[0] == '(', sexpr
sexpr
=
sexpr
[
1
:].
strip
()
m
=
re
.
match
(
'[\w:]+'
,
sexpr
)
name
,
class_name
=
(
sexpr
[:
m
.
end
()].
split
(
':'
)
+
[
''
])[:
2
]
sexpr
=
sexpr
[
m
.
end
():].
strip
()
if
sexpr
[
0
]
==
'('
:
result
=
tuple
(
mock_syntax_tree
(
block
)
for
block
in
next_block
(
sexpr
))
else
:
lines
=
[]
while
sexpr
and
sexpr
[
0
]
!=
')'
:
for
qm
in
[
'"""'
,
"'''"
,
'"'
,
"'"
]:
m
=
re
.
match
(
qm
+
r
'.*?'
+
qm
,
sexpr
)
if
m
:
i
=
len
(
qm
)
lines
.
append
(
sexpr
[
i
:
m
.
end
()
-
i
])
sexpr
=
sexpr
[
m
.
end
():].
strip
()
break
else
:
m
=
re
.
match
(
r
'(?:(?!\)).)*'
,
sexpr
)
lines
.
append
(
sexpr
[:
m
.
end
()])
sexpr
=
sexpr
[
m
.
end
():]
result
=
"
\n
"
.
join
(
lines
)
return
Node
(
MockParser
(
name
,
':'
+
class_name
),
result
)
def
recompile_grammar
(
ebnf_filename
,
force
=
False
)
->
bool
:
"""
Recompiles an ebnf-grammar if necessary, that is if either no
corresponding 'XXXXCompiler.py'-file exists or if that file is
outdated.
Parameters:
ebnf_filename(str): The filename of the ebnf-source of the
grammar. In case this is a directory and not a file all
files within this directory ending with .ebnf will be
compiled.
force(bool): If False (default), the grammar will only be
recompiled if it has been changed.
"""
if
os
.
path
.
isdir
(
ebnf_filename
):
success
=
True
for
entry
in
os
.
listdir
(
ebnf_filename
):
if
entry
.
lower
().
endswith
(
'.ebnf'
)
and
os
.
path
.
isfile
(
entry
):
success
=
success
and
recompile_grammar
(
entry
,
force
)
return
success
base
,
ext
=
os
.
path
.
splitext
(
ebnf_filename
)
compiler_name
=
base
+
'Compiler.py'
error_file_name
=
base
+
'_ebnf_ERRORS.txt'
errors
=
[]
if
(
not
os
.
path
.
exists
(
compiler_name
)
or
force
or
grammar_changed
(
compiler_name
,
ebnf_filename
)):
# print("recompiling parser for: " + ebnf_filename)
errors
=
compile_on_disk
(
ebnf_filename
)
if
errors
:
# print("Errors while compiling: " + ebnf_filename + '!')
with
open
(
error_file_name
,
'w'
)
as
f
:
for
e
in
errors
:
f
.
write
(
e
)
f
.
write
(
'
\n
'
)
return
False
if
not
errors
and
os
.
path
.
exists
(
error_file_name
):
os
.
remove
(
error_file_name
)
return
True
UNIT_STAGES
=
{
'match'
,
'fail'
,
'ast'
,
'cst'
,
'__ast__'
,
'__cst__'
}
...
...
DHParser/toolkit.py
View file @
da2df627
...
...
@@ -45,8 +45,7 @@ try:
except
ImportError
:
from
.typing34
import
Any
,
List
,
Tuple
__all__
=
[
'logging'
,
__all__
=
(
'logging'
,
'is_logging'
,
'log_dir'
,
'logfile_basename'
,
...
...
@@ -63,7 +62,7 @@ __all__ = ['logging',
'md5'
,
'expand_table'
,
'smart_list'
,
'sane_parser_name'
]
'sane_parser_name'
)
def
log_dir
()
->
str
:
...
...
DHParser/transform.py
View file @
da2df627
...
...
@@ -52,6 +52,7 @@ __all__ = ('transformation_factory',
'is_token'
,
'has_name'
,
'has_content'
,
'remove_children_if'
,
'remove_parser'
,
'remove_content'
,
'remove_first'
,
...
...
@@ -61,6 +62,7 @@ __all__ = ('transformation_factory',
'remove_expendables'
,
'remove_brackets'
,
'remove_tokens'
,
'keep_children'
,
'flatten'
,
'forbid'
,
'require'
,
...
...
DHParser/versionnumber.py
View file @
da2df627
...
...
@@ -16,5 +16,5 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
__all__
=
(
'__version__'
)
__version__
=
'0.7.6'
# + '_dev' + str(os.stat(__file__).st_mtime)
__all__
=
[
'__version__'
]
DevScripts/Readme-DevScripts.md
View file @
da2df627
...
...
@@ -3,3 +3,5 @@ Folder "DevScripts"
This folder contains helper scripts for the
development of DHParser.
collect_symbols.py - Lists all exported symbols from DHParser modules
DevScripts/collect_symbols.py
View file @
da2df627
"""collect_symbols.py -
EBNF ->
Lists all exported symbols from DHParser
"""collect_symbols.py - Lists all exported symbols from DHParser
modules
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
...
...
DevScripts/create_standalone.py
0 → 100644
View file @
da2df627
"""create_standalone.py - merges the DHParser modules into a standalone
DHParser.py module for easier deployment.
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import
functools
import
operator
import
sys
sys
.
path
.
append
(
'../'
)
try
:
import
regex
as
re
except
ImportError
:
import
re
modules
=
(
'toolkit'
,
'syntaxtree'
,
'parser'
,
'transform'
,
'ebnf'
,
'dsl'
,
'testing'
,
'versionnumber'
)
all_symbols
=
list
(
functools
.
reduce
(
operator
.
or_
,
(
set
(
eval
(
m
+
'.__all__'
))
for
m
in
modules
)))
all_symbols
.
sort
()
def
start
(
module
):
i
=
module
.
find
(
'__all__'
)
i
=
module
.
find
(
')'
,
i
)
i
=
module
.
find
(
'
\n
'
,
i
)
+
1
return
i
doc
=
"DHParser.py - Packrat-parser and parser-generator
\n\n
"
+
__doc__
[
__doc__
.
find
(
'Copyright'
):]
imports
=
"""
import abc
import codecs
import collections
from collections import OrderedDict
import configparser
import contextlib
import copy
from functools import partial
import hashlib
import inspect
import json
import os
import platform
try:
import regex as re
except ImportError:
import re
import sys
from typing import AbstractSet, Any, ByteString, Callable, cast, Container, Dict,
\\
Iterator, List, NamedTuple, Sequence, Set, Union, Text, Tuple
"""
heading
=
"""
#######################################################################
#######################################################################
#
# %s
#
#######################################################################
#######################################################################
"""
def
merge_modules
(
module_names
,
dhp_path
=
'../DHParser/'
):
components
=
[
doc
,
imports
]
for
name
in
module_names
:
with
open
(
dhp_path
+
'%s.py'
%
name
)
as
f
:
module
=
f
.
read
()
content
=
module
[
start
(
module
):]
components
.
append
(
heading
%
name
)
components
.
append
(
content
)
return
"
\n
"
.
join
(
components
)
print
(
merge_modules
(
modules
))
examples/LaTeX/recompile_grammar_obsolote.py
View file @
da2df627
...
...
@@ -20,14 +20,13 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import
cProfile
import
functools
import
sys
sys
.
path
.
extend
([
'../../'
,
'../'
,
'./'
])
from
DHParser.toolkit
import
logging
from
DHParser.
testing
import
recompile_grammar
from
DHParser.
dsl
import
recompile_grammar
def
profile
(
func
):
...
...
examples/LaTeX/tst_grammar.py
View file @
da2df627
...
...
@@ -20,10 +20,14 @@ limitations under the License.
"""
import
sys
import
DHParser.dsl
sys
.
path
.
extend
([
'../../'
,
'../'
,
'./'
])
from
DHParser
import
testing
if
not
testing
.
recompile_grammar
(
'LaTeX.ebnf'
,
force
=
True
):
# recompiles Grammar only if it has changed
if
not
DHParser
.
dsl
.
recompile_grammar
(
'LaTeX.ebnf'
,
force
=
True
):
# recompiles Grammar only if it has changed
with
open
(
'LaTeX_ebnf_ERRORS.txt'
)
as
f
:
print
(
f
.
read
())
sys
.
exit
(
1
)
...
...
examples/MLW/recompile_grammar.py
View file @
da2df627
...
...
@@ -20,7 +20,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
from
DHParser.testing
import
recompile_grammar
from
DHParser.dsl
import
recompile_grammar
recompile_grammar
(
'.'
)
# import os
...
...
examples/Tutorial/recompile_grammar.py
View file @
da2df627
...
...
@@ -26,7 +26,7 @@ import sys
sys
.
path
.
extend
([
'../../'
,
'../'
,
'./'
])
flag
=
os
.
path
.
exists
(
'LyrikCompiler.py'
)
from
DHParser.
testing
import
recompile_grammar
from
DHParser.
dsl
import
recompile_grammar
if
not
recompile_grammar
(
'.'
,
force
=
True
):
with
open
(
'Lyrik_ebnf_ERRORS.txt'
)
as
f
:
...
...
test/test_syntaxtree.py
View file @
da2df627
...
...
@@ -23,10 +23,9 @@ import copy
import
sys
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser.syntaxtree
import
Node
,
TOKEN_PTYPE
from
DHParser.syntaxtree
import
Node
,
TOKEN_PTYPE
,
mock_syntax_tree
from
DHParser.transform
import
traverse
,
reduce_single_child
,
\
replace_by_single_child
,
flatten
,
remove_expendables
from
DHParser.testing
import
mock_syntax_tree
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
from
DHParser.dsl
import
parser_factory
...
...
test/test_testing.py
View file @
da2df627
...
...
@@ -25,11 +25,11 @@ from functools import partial
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser.toolkit
import
compact_sexpr
from
DHParser.syntaxtree
import
TOKEN_PTYPE
from
DHParser.syntaxtree
import
TOKEN_PTYPE
,
mock_syntax_tree
from
DHParser.transform
import
traverse
,
remove_expendables
,
\
replace_by_single_child
,
reduce_single_child
,
flatten
from
DHParser.dsl
import
parser_factory
from
DHParser.testing
import
grammar_unit
,
mock_syntax_tree
from
DHParser.testing
import
grammar_unit
ARITHMETIC_EBNF
=
"""
@ whitespace = linefeed
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment