Skip to content
GitLab
Commit 524501e5 authored by eckhart's avatar eckhart
Browse files

- some more optimizations

parent 0c1fbc00
Hide whitespace changes
Inline Side-by-side
DHParser/parse.pxd
View file @ 524501e5
......@@ -11,6 +11,8 @@ cdef class Parser:
cdef object recursion_counter
cdef object cycle_detection
cpdef _return_node(self, node)
cpdef _return_node_from_results(self, results)
cpdef _parse(self, text)
cpdef reset(self)
cpdef _apply(self, func, flip)
......
DHParser/parse.py
View file @ 524501e5
......@@ -55,8 +55,8 @@ __all__ = ('Parser',
'Whitespace',
'DropWhitespace',
'mixin_comment',
'UnaryOperator',
'NaryOperator',
'UnaryParser',
'NaryParser',
'Synonym',
'Option',
'ZeroOrMore',
......@@ -380,6 +380,30 @@ class Parser:
"""
return Alternative(self, other)
def _return_node(self, node: Node) -> Node:
# Node(self.tag_name, node) # unoptimized code
if node and node._result:
return Node(self.tag_name, node) if self.pname else node
if self.pname:
return Node(self.tag_name, ())
else:
# avoid creation of a node object for empty nodes
return EMPTY_NODE
@cython.locals(N=cython.int)
def _return_node_from_results(self, results: Tuple[Node, ...]) -> Node:
# return Node(self.tag_name, results) # unoptimized code
N = len(results)
if N > 1:
return Node(self.tag_name, results)
elif N == 1:
return self._return_node(results[0])
elif self.pname:
return Node(self.tag_name, ())
else:
# avoid creation of a node object for empty nodes
return EMPTY_NODE
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
"""Applies the parser to the given `text` and returns a node with
the results or None as well as the text at the position right behind
......@@ -1237,19 +1261,19 @@ class DropWhitespace(Whitespace):
########################################################################
class UnaryOperator(Parser):
class UnaryParser(Parser):
"""
Base class of all unary parser operators, i.e. parser that contains
Base class of all unary parsers, i.e. parser that contains
one and only one other parser, like the optional parser for example.
The UnaryOperator base class supplies __deepcopy__ and apply
methods for unary parser operators. The __deepcopy__ method needs
methods for unary parsers. The __deepcopy__ method needs
to be overwritten, however, if the constructor of a derived class
has additional parameters.
"""
def __init__(self, parser: Parser) -> None:
super(UnaryOperator, self).__init__()
super(UnaryParser, self).__init__()
assert isinstance(parser, Parser), str(parser)
self.parser = parser # type: Parser
......@@ -1261,26 +1285,26 @@ class UnaryOperator(Parser):
return duplicate
def _apply(self, func: ApplyFunc, flip: FlagFunc) -> bool:
if super(UnaryOperator, self)._apply(func, flip):
if super(UnaryParser, self)._apply(func, flip):
self.parser._apply(func, flip)
return True
return False
class NaryOperator(Parser):
class NaryParser(Parser):
"""
Base class of all Nnary parser operators, i.e. parser that
Base class of all Nnary parsers, i.e. parser that
contains one or more other parsers, like the alternative
parser for example.
The NnaryOperator base class supplies __deepcopy__ and apply methods
for unary parser operators. The __deepcopy__ method needs to be
for unary parsers. The __deepcopy__ method needs to be
overwritten, however, if the constructor of a derived class has
additional parameters.
"""
def __init__(self, *parsers: Parser) -> None:
super(NaryOperator, self).__init__()
super(NaryParser, self).__init__()
assert all([isinstance(parser, Parser) for parser in parsers]), str(parsers)
self.parsers = parsers # type: Tuple[Parser, ...]
......@@ -1292,14 +1316,14 @@ class NaryOperator(Parser):
return duplicate
def _apply(self, func: ApplyFunc, flip: FlagFunc) -> bool:
if super(NaryOperator, self)._apply(func, flip):
if super(NaryParser, self)._apply(func, flip):
for parser in self.parsers:
parser._apply(func, flip)
return True
return False
class Option(UnaryOperator):
class Option(UnaryParser):
r"""
Parser ``Option`` always matches, even if its child-parser
did not match.
......@@ -1318,7 +1342,7 @@ class Option(UnaryOperator):
>>> Grammar(number)('3.14159').content
'3.14159'
>>> Grammar(number)('3.14159').structure
'(:Series (:RegExp "3") (:Option (:RegExp ".14159")))'
'(:Series (:RegExp "3") (:RegExp ".14159"))'
>>> Grammar(number)('-1').content
'-1'
......@@ -1335,13 +1359,7 @@ class Option(UnaryOperator):
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
node, text = self.parser(text)
if node and (node._result or self.parser.pname):
return Node(self.tag_name, node), text
if self.pname:
return Node(self.tag_name, ()), text
else:
# avoid creation of a node object for empty nodes
return EMPTY_NODE, text
return self._return_node(node), text
def __repr__(self):
return '[' + (self.parser.repr[1:-1] if isinstance(self.parser, Alternative)
......@@ -1382,7 +1400,7 @@ class ZeroOrMore(Option):
infinite_loop_error = Error(dsl_error_msg(self, 'Infinite Loop encountered.'),
node.pos)
results += (node,)
node = Node(self.tag_name, results)
node = self._return_node_from_results(results) # type: Node
if infinite_loop_error:
self.grammar.tree__.add_error(node, infinite_loop_error)
return node, text
......@@ -1392,7 +1410,7 @@ class ZeroOrMore(Option):
and not self.parser.pname else self.parser.repr) + '}'
class OneOrMore(UnaryOperator):
class OneOrMore(UnaryParser):
r"""
`OneOrMore` applies a parser repeatedly as long as this parser
matches. Other than `ZeroOrMore` which always matches, at least
......@@ -1434,7 +1452,7 @@ class OneOrMore(UnaryOperator):
results += (node,)
if results == ():
return None, text
node = Node(self.tag_name, results)
node = self._return_node_from_results(results) # type: Node
if infinite_loop_error:
self.grammar.tree__.add_error(node, infinite_loop_error)
return node, text_
......@@ -1476,7 +1494,7 @@ def mandatory_violation(grammar: Grammar,
return error, err_node, text_[i:]
class Series(NaryOperator):
class Series(NaryParser):
r"""
Matches if each of a series of parsers matches exactly in the order of
the series.
......@@ -1565,7 +1583,7 @@ class Series(NaryOperator):
results += (node,)
# assert len(results) <= len(self.parsers) \
# or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
node = Node(self.tag_name, results)
node = self._return_node_from_results(results) # type: Node
if error:
raise ParserError(node, text, first_throw=True)
return node, text_
......@@ -1613,7 +1631,7 @@ class Series(NaryOperator):
return self
class Alternative(NaryOperator):
class Alternative(NaryParser):
r"""
Matches if one of several alternatives matches. Returns
the first match.
......@@ -1679,7 +1697,7 @@ class Alternative(NaryOperator):
return self
class AllOf(NaryOperator):
class AllOf(NaryParser):
"""
Matches if all elements of a list of parsers match. Each parser must
match exactly once. Other than in a sequence, the order in which
......@@ -1777,7 +1795,7 @@ class AllOf(NaryOperator):
parsers = []
assert len(results) <= len(self.parsers) \
or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
node = Node(self.tag_name, results)
node = self._return_node_from_results(results) # type: Node
if error:
raise ParserError(node, text, first_throw=True)
return node, text_
......@@ -1786,7 +1804,7 @@ class AllOf(NaryOperator):
return '< ' + ' '.join(parser.repr for parser in self.parsers) + ' >'
class SomeOf(NaryOperator):
class SomeOf(NaryParser):
"""
Matches if at least one element of a list of parsers match. No parser
must match more than once . Other than in a sequence, the order in which
......@@ -1833,7 +1851,7 @@ class SomeOf(NaryOperator):
parsers = []
assert len(results) <= len(self.parsers)
if results:
return Node(self.tag_name, results), text_
return self._return_node_from_results(results), text_
else:
return None, text
......@@ -1841,7 +1859,7 @@ class SomeOf(NaryOperator):
return '< ' + ' | '.join(parser.repr for parser in self.parsers) + ' >'
def Unordered(parser: NaryOperator) -> NaryOperator:
def Unordered(parser: NaryParser) -> NaryParser:
"""
Returns an AllOf- or SomeOf-parser depending on whether `parser`
is a Series (AllOf) or an Alternative (SomeOf).
......@@ -1856,13 +1874,13 @@ def Unordered(parser: NaryOperator) -> NaryOperator:
########################################################################
#
# Flow control operators
# Flow control parsers
#
########################################################################
class FlowOperator(UnaryOperator):
class FlowParser(UnaryParser):
"""
Base class for all flow operator parsers like Lookahead and Lookbehind.
Base class for all flow parsers like Lookahead and Lookbehind.
"""
def sign(self, bool_value) -> bool:
"""Returns the value. Can be overriden to return the inverted bool."""
......@@ -1873,7 +1891,7 @@ def Required(parser: Parser) -> Parser:
return Series(parser, mandatory=0)
# class Required(FlowOperator):
# class Required(FlowParser):
# """OBSOLETE. Use mandatory-parameter of Series-parser instead!
# """
# RX_ARGUMENT = re.compile(r'\s(\S)')
......@@ -1894,7 +1912,7 @@ def Required(parser: Parser) -> Parser:
# return '§' + self.parser.repr
class Lookahead(FlowOperator):
class Lookahead(FlowParser):
"""
Matches, if the contained parser would match for the following text,
but does not consume any text.
......@@ -1922,7 +1940,7 @@ class NegativeLookahead(Lookahead):
return not bool_value
class Lookbehind(FlowOperator):
class Lookbehind(FlowParser):
"""
Matches, if the contained parser would match backwards. Requires
the contained parser to be a RegExp, _RE, PlainText or _Token parser.
......@@ -1968,12 +1986,12 @@ class NegativeLookbehind(Lookbehind):
########################################################################
#
# Capture and Retrieve operators (for passing variables in the parser)
# Capture and Retrieve parsers (for passing variables in the parser)
#
########################################################################
class Capture(UnaryOperator):
class Capture(UnaryParser):
"""
Applies the contained parser and, in case of a match, saves the result
in a variable. A variable is a stack of values associated with the
......@@ -1991,7 +2009,7 @@ class Capture(UnaryOperator):
self.grammar.push_rollback__(location, self._rollback) # lambda: stack.pop())
# caching will be blocked by parser guard (see way above),
# because it would prevent recapturing of rolled back captures
return Node(self.tag_name, node), text_
return self._return_node(node), text_
else:
return None, text
......@@ -2121,7 +2139,7 @@ class Pop(Retrieve):
########################################################################
class Synonym(UnaryOperator):
class Synonym(UnaryParser):
r"""
Simply calls another parser and encapsulates the result in
another node if that parser matches.
......
examples/Arithmetic/Arithmetic.ebnf 0 → 100644
View file @ 524501e5
# Arithmetic-grammar
#######################################################################
#
# EBNF-Directives
#
#######################################################################
@ whitespace = vertical # implicit whitespace, includes any number of line feeds
@ literalws = right # literals have implicit whitespace on the right hand side
@ comment = /#.*/ # comments range from a '#'-character to the end of the line
@ ignorecase = False # literals and regular expressions are case-sensitive
@ drop = whitespace # drop anonymous whitespace and tokens
#######################################################################
#
# Structure and Components
#
#######################################################################
expression = term { ("+" | "-") term}
term = factor { ("*" | "/") factor}
factor = [/-/] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
group = "(" expression ")"
#######################################################################
#
# Regular Expressions
#
#######################################################################
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
examples/Arithmetic/ArithmeticCompiler.py 0 → 100755
View file @ 524501e5
#!/usr/bin/python
#######################################################################
#
# SYMBOLS SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
import collections
from functools import partial
import os
import sys
sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, PreprocessorToken, Whitespace, DropWhitespace, \
Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \
Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
remove_children_if, move_whitespace, normalize_whitespace, is_anonymous, matches_re, \
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, is_empty, \
is_expendable, collapse, collapse_if, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
remove_nodes, remove_content, remove_brackets, replace_parser, remove_anonymous_tokens, \
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
error_on, recompile_grammar, GLOBALS
#######################################################################
#
# PREPROCESSOR SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
def ArithmeticPreprocessor(text):
return text, lambda i: i
def get_preprocessor() -> PreprocessorFunc:
return ArithmeticPreprocessor
#######################################################################
#
# PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!
#
#######################################################################
class ArithmeticGrammar(Grammar):
r"""Parser for an Arithmetic source file.
"""
expression = Forward()
source_hash__ = "d03e397fb4cabd6f20f3ae7c9add4ad5"
parser_initialization__ = ["upon instantiation"]
resume_rules__ = {}
COMMENT__ = r'#.*'
WHITESPACE__ = r'\s*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
dwsp__ = DropWhitespace(WSP_RE__)
wsp__ = Whitespace(WSP_RE__)
VARIABLE = Series(RegExp('[A-Za-z]'), dwsp__)
NUMBER = Series(RegExp('(?:0|(?:[1-9]\\d*))(?:\\.\\d+)?'), dwsp__)
group = Series(Series(Token("("), dwsp__), expression, Series(Token(")"), dwsp__))
factor = Series(Option(RegExp('-')), Alternative(NUMBER, VARIABLE, group), ZeroOrMore(Alternative(VARIABLE, group)))
term = Series(factor, ZeroOrMore(Series(Alternative(Series(Token("*"), dwsp__), Series(Token("/"), dwsp__)), factor)))
expression.set(Series(term, ZeroOrMore(Series(Alternative(Series(Token("+"), dwsp__), Series(Token("-"), dwsp__)), term))))
root__ = expression
def get_grammar() -> ArithmeticGrammar:
global GLOBALS
try:
grammar = GLOBALS.Arithmetic_00000001_grammar_singleton
except AttributeError:
GLOBALS.Arithmetic_00000001_grammar_singleton = ArithmeticGrammar()
if hasattr(get_grammar, 'python_src__'):
GLOBALS.Arithmetic_00000001_grammar_singleton.python_src__ = get_grammar.python_src__
grammar = GLOBALS.Arithmetic_00000001_grammar_singleton
return grammar
#######################################################################
#
# AST SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
Arithmetic_AST_transformation_table = {
# AST Transformations for the Arithmetic-grammar
"<": remove_empty,
"expression": [],
"term": [],
"factor": [replace_or_reduce],
"NUMBER": [],
"VARIABLE": [],
":Token": reduce_single_child,
"*": replace_by_single_child
}
def ArithmeticTransform() -> TransformationDict:
return partial(traverse, processing_table=Arithmetic_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
try:
transformer = GLOBALS.Arithmetic_00000001_transformer_singleton
except AttributeError:
GLOBALS.Arithmetic_00000001_transformer_singleton = ArithmeticTransform()
transformer = GLOBALS.Arithmetic_00000001_transformer_singleton
return transformer
#######################################################################
#
# COMPILER SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
class ArithmeticCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a Arithmetic source file.
"""
def __init__(self):
super(ArithmeticCompiler, self).__init__()
def _reset(self):
super()._reset()
# initialize your variables here, not in the constructor!
def on_expression(self, node):
return self.fallback_compiler(node)
# def on_term(self, node):
# return node
# def on_factor(self, node):
# return node
# def on_NUMBER(self, node):
# return node
# def on_VARIABLE(self, node):
# return node
def get_compiler() -> ArithmeticCompiler:
try:
compiler = GLOBALS.Arithmetic_00000001_compiler_singleton
except AttributeError:
GLOBALS.Arithmetic_00000001_compiler_singleton = ArithmeticCompiler()
compiler = GLOBALS.Arithmetic_00000001_compiler_singleton
return compiler
#######################################################################
#
# END OF DHPARSER-SECTIONS
#
#######################################################################
def compile_src(source, log_dir=''):
"""Compiles ``source`` and returns (result, errors, ast).
"""
with logging(log_dir):
compiler = get_compiler()
cname = compiler.__class__.__name__
result_tuple = compile_source(source, get_preprocessor(),
get_grammar(),
get_transformer(), compiler)
return result_tuple
if __name__ == "__main__":
# recompile grammar if needed
grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
if os.path.exists(grammar_path):
if not recompile_grammar(grammar_path, force=False,
notify=lambda:print('recompiling ' + grammar_path)):
error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
with open(error_file, encoding="utf-8") as f:
print(f.read())
sys.exit(1)
else:
print('Could not check whether grammar requires recompiling, '
'because grammar was not found at: ' + grammar_path)
if len(sys.argv) > 1:
# compile file
file_name, log_dir = sys.argv[1], ''
if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
file_name, log_dir = sys.argv[2], 'LOGS'
result, errors, ast = compile_src(file_name, log_dir)
if errors:
cwd = os.getcwd()
rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
for error in errors:
print(rel_path + ':' + str(error))
sys.exit(1)
else:
print(result.as_xml() if isinstance(result, Node) else result)
else:
print("Usage: ArithmeticCompiler.py [FILENAME]")
examples/Arithmetic/README.md 0 → 100644
View file @ 524501e5
# Arithmetic
PLACE A SHORT DESCRIPTION HERE
Author: AUTHOR'S NAME <EMAIL>, AFFILIATION
## License
Arithmetic is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
examples/Arithmetic/example.dsl 0 → 100644
View file @ 524501e5
Life is but a walking shadow
examples/Arithmetic/grammar_tests/01_test_REGUALAR_EXPRESSIONS.ini 0 → 100644
View file @ 524501e5
[match:NUMBER]
M1: 1
M2: 99
M3: 3.14156
M5: 2
M6: 105.23
[fail:NUMBER]
F1: +22
F2: 003
F4: 2.2.6
F5: -22
[match:VARIABLE]
M1: a
M2: Z
[fail:VARIABLE]
F1: ä
F2: ab
examples/Arithmetic/grammar_tests/02_test_components.ini 0 → 100644
View file @ 524501e5
[match:group]
M1*: "(2 + x)"
M2: "(3)"
[fail:group]
F1: "22"
F2: "y"
<