Commit c124120f authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

ebnf.py: "@literalws" default is now "none"

parent f4572d14
......@@ -329,6 +329,15 @@ CONFIG_PRESET['add_grammar_source_to_parser_docstring'] = False
CONFIG_PRESET['default_anonymous_regexp'] = r'..(?<=^)'
# Default value for implicit insignificant whitespace adjacent to literals.
# Possible values are:
# 'none': no implicit adjacent whitespace: "text" = `text`
# 'right': implicit whitespace to the right: "text" = `text`~
# 'left': implicit whitespace to the left: "text" = ~`text`
# 'both': implicit whitespace on both sides: "text" = ~`text`~
CONFIG_PRESET['default_literalws'] = "none"
# Default value for the brand of EBNF that DHParser accepts
# 'classic' - relatively closest to the ISO-standard, i.e. uses [] and {}
# for optional and zero or more elements, respectively. Does not allow
......
......@@ -749,7 +749,7 @@ class EBNFDirectives:
def __init__(self):
self.whitespace = WHITESPACE_TYPES['vertical'] # type: str
self.comment = '' # type: str
self.literalws = {'none'} # type: Set[str]
self.literalws = {get_config_value('default_literalws')} # type: Set[str]
self.tokens = set() # type: Set[str]
self.filter = dict() # type: Dict[str, str]
self.error = dict() # type: Dict[str, List[Tuple[ReprType, ReprType]]]
......
......@@ -35,6 +35,7 @@ from DHParser.dsl import compile_on_disk, run_compiler, compileEBNF, grammar_pro
from DHParser.toolkit import concurrent_ident
ARITHMETIC_EBNF = """
@ literalws = right
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
......
......@@ -45,6 +45,7 @@ from DHParser.testing import grammar_unit, clean_report
class TestDirectives:
mini_language = """
@ literalws = right
expression = term { ("+" | "-") term }
term = factor { ("*" | "/") factor }
factor = constant | "(" expression ")"
......@@ -489,7 +490,7 @@ class TestWhitespace:
WORD = /\w+/~
EOF = !/./
"""
lang1 = r'document = "DOC" { WORD } EOF' + tail
lang1 = '@literalws=right\ndocument = "DOC" { WORD } EOF' + tail
parser = grammar_provider(lang1)()
cst = parser("DOC Wörter Wörter Wörter")
assert not cst.error_flag
......@@ -766,7 +767,7 @@ class TestErrorCustomizationErrors:
class TestCustomizedResumeParsing:
lang = r"""
lang = r"""@ literalws = right
@ alpha_resume = "BETA", "GAMMA"
@ beta_resume = GAMMA_RE
@ bac_resume = /(?=GA\w+)/
......@@ -970,6 +971,7 @@ class TestInterleaveResume:
ArithmeticEBNF = r"""
@ literalws = right
@ drop = whitespace # <- there is no alternative syntax for directives!!!
expression ::= term, { ("+" | "-"), term};
......
......@@ -112,7 +112,7 @@ class TestParserClass:
class TestInfiLoopsAndRecursion:
def test_direct_left_recursion1(self):
minilang = """
minilang = """@literalws = right
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~
......@@ -128,7 +128,7 @@ class TestInfiLoopsAndRecursion:
log_parsing_history(parser, "test_LeftRecursion_direct")
def test_direct_left_recursion2(self):
minilang = """
minilang = """@literalws = right
expr = ex
ex = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
......@@ -142,7 +142,7 @@ class TestInfiLoopsAndRecursion:
assert snippet == syntax_tree.content
def test_indirect_left_recursion1(self):
minilang = """
minilang = """@literalws = right
Expr = //~ (Product | Sum | Value)
Product = Expr { ('*' | '/') Expr }+
Sum = Expr { ('+' | '-') Expr }+
......@@ -165,7 +165,7 @@ class TestInfiLoopsAndRecursion:
# # BEWARE: EXPERIMENTAL TEST can be long running
# def test_indirect_left_recursion2(self):
# arithmetic_syntax = """
# arithmetic_syntax = """@literalws = right
# expression = addition | subtraction
# addition = (expression | term) "+" (expression | term)
# subtraction = (expression | term) "-" (expression | term)
......@@ -363,7 +363,7 @@ class TestRegex:
assert node is None
def test_token(self):
tokenlang = r"""
tokenlang = r"""@literalws = right
@whitespace = linefeed
lang = "" begin_token {/\w+/ ""} end_token
begin_token = "\begin{document}"
......@@ -667,7 +667,7 @@ class TestPopRetrieve:
closing_braces = /\}+/
text = /[^{}]+/
"""
mini_lang3 = r"""
mini_lang3 = r"""@literalws = right
document = { text | env }
env = (specialtag | opentag) text [ closespecial | closetag ]
opentag = "<" name ">"
......@@ -679,7 +679,7 @@ class TestPopRetrieve:
name = /\w+/~
text = /[^<>]+/
"""
mini_lang4 = r"""
mini_lang4 = r"""@literalws = right
document = { text | env }
env = opentag document closetag
opentag = "<" name ">"
......@@ -770,7 +770,7 @@ class TestPopRetrieve:
def test_cache_neutrality(self):
"""Test that packrat-caching does not interfere with the variable-
changing parsers: Capture and Retrieve."""
lang = r"""
lang = r"""@literalws = right
text = opening closing
opening = (unmarked_package | marked_package)
closing = ::variable
......@@ -845,7 +845,7 @@ class TestPopRetrieve:
log_ST(syntax_tree, "test_PopRetrieve_multi_line.cst")
def test_autoretrieve(self):
lang = r"""
lang = r"""@literalws = right
document = { definition } § EOF
definition = symbol :defsign value
symbol = /\w+/~
......@@ -864,15 +864,15 @@ class TestPopRetrieve:
lines = [line for line in lang.split('\n') if line.strip()]
eof_line = lines.pop()
lines.insert(1, eof_line)
lines.insert(2, eof_line)
lang = '\n'.join(lines)
parser = grammar_provider(lang)()
st = parser("X := 1")
assert not st.errors
assert not st.errors, str(st.errors)
assert st.equals(st1)
del lines[1]
lines.insert(2, eof_line)
del lines[2]
lines.insert(3, eof_line)
lang = '\n'.join(lines)
parser = grammar_provider(lang)()
st = parser("X := 1")
......@@ -880,7 +880,7 @@ class TestPopRetrieve:
assert st.equals(st1)
# and, finally...
lang_variant = r"""
lang_variant = r"""@literalws = right
document = { definition } § EOF
symbol = /\w+/~
defsign = "=" | ":="
......@@ -897,7 +897,7 @@ class TestPopRetrieve:
class TestWhitespaceHandling:
minilang = """
minilang = """@literalws = right
doc = A B
A = "A"
B = "B"
......@@ -1049,7 +1049,7 @@ EOF = !/./ [:?DEF] [:?OR] [:?AND] [:?ENDL]
"""
class TestReentryAfterError:
testlang = """
testlang = """@literalws = right
document = alpha [beta] gamma "."
alpha = "ALPHA" abc
abc = §"a" "b" "c"
......@@ -1167,7 +1167,7 @@ def next_valid_letter(text, start):
def test_skip_comment_on_resume(self):
lang = r"""
lang = r"""@literalws = right
@ comment = /(?:\/\/.*)|(?:\/\*(?:.|\n)*?\*\/)/ # Kommentare im C++-Stil
document = block_A block_B
@ block_A_resume = /(?=x)/
......@@ -1191,6 +1191,7 @@ def next_valid_letter(text, start):
def test_unambiguous_error_location(self):
lang = r"""
@ literalws = right
@ drop = whitespace, strings # drop strings and whitespace early
@object_resume = /(?<=\})/
......
......@@ -296,7 +296,7 @@ class TestNode:
assert parse_sxpr('(a (b c))').equals(parse_sxpr('(a (b c))'))
def test_equality2(self):
ebnf = 'term = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~'
ebnf = '@literalws = right\nterm = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~'
att = {"term": [remove_empty, remove_whitespace, replace_by_single_child, flatten],
"factor": [remove_empty, remove_whitespace, reduce_single_child],
"*": [remove_empty, remove_whitespace, replace_by_single_child]}
......
......@@ -67,6 +67,7 @@ class TestTrace:
def test_trace_simple(self):
lang = """
@literalws = right
expr = term { ("+"|"-") term }
term = factor { ("*"|"/") factor }
factor = /[0-9]+/~ | "(" expr ")"
......@@ -89,6 +90,7 @@ class TestTrace:
def test_trace_stopped_early(self):
lang = """
@ literalws = right
expr = term { ("+"|"-") term }
term = factor { ("*"|"/") factor }
factor = /[0-9]+/~ | "(" expr ")"
......@@ -103,6 +105,7 @@ class TestTrace:
def test_trace_drop(self):
lang = r"""
@ literalws = right
@ drop = strings, whitespace
expression = term { ("+" | "-") term}
term = factor { ("*"|"/") factor}
......@@ -129,6 +132,7 @@ class TestTrace:
def test_trace_resume(self):
lang = """
@literalws = right
document = alpha [beta] gamma "."
alpha = "ALPHA" abc
abc = §"a" "b" "c"
......@@ -161,6 +165,7 @@ class TestTrace:
class TestErrorReporting:
lang = """
@literalws = right
document = alpha [beta] gamma "."
alpha = "ALPHA" abc
abc = §"a" "b" "c"
......@@ -233,6 +238,7 @@ class TestErrorReporting:
def test_trace_resume_complex_case(self):
lang = r"""
@ literalws = right
@ comment = /(?:\/\/.*)|(?:\/\*(?:.|\n)*?\*\/)/ # Kommentare im C++-Stil
document = block_A block_B
@ block_A_resume = /(?=x)/
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment