10.12., 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit da2df627 authored by Eckhart Arnold's avatar Eckhart Arnold

- script for creating single-module ("stand-alone") version of DHParser

parent 98411756
......@@ -24,6 +24,7 @@ from .parser import *
from .syntaxtree import *
# Flat namespace for the DHParser Package. Is this a good idea...?
from .toolkit import *
from .transform import *
# from .testing import *
from .versionnumber import __version__
......
......@@ -20,6 +20,7 @@ compilation of domain specific languages based on an EBNF-grammar.
"""
import os
try:
import regex as re
except ImportError:
......@@ -36,15 +37,14 @@ from DHParser.toolkit import logging, load_if_file, is_python_code, compile_pyth
from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, TransformationFunc
__all__ = ['GrammarError',
__all__ = ('GrammarError',
'CompilationError',
'load_compiler_suite',
'compileDSL',
'raw_compileEBNF',
'compileEBNF',
'parser_factory',
'compile_on_disk']
'compile_on_disk')
SECTION_MARKER = """\n
......@@ -74,14 +74,14 @@ try:
import regex as re
except ImportError:
import re
from DHParser.toolkit import logging, is_filename, load_if_file
from DHParser.parser import Grammar, Compiler, nil_preprocessor, \\
from DHParser import logging, is_filename, load_if_file, \\
Grammar, Compiler, nil_preprocessor, \\
Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
last_value, counterpart, accumulate, PreprocessorFunc
from DHParser.syntaxtree import Node, TransformationFunc
from DHParser.transform import traverse, remove_children_if, \\
last_value, counterpart, accumulate, PreprocessorFunc, \\
Node, TransformationFunc, \\
traverse, remove_children_if, \\
reduce_single_child, replace_by_single_child, remove_whitespace, \\
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
......@@ -483,3 +483,45 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
if f: f.close()
return []
def recompile_grammar(ebnf_filename, force=False) -> bool:
"""
Recompiles an ebnf-grammar if necessary, that is, if either no
corresponding 'XXXXCompiler.py'-file exists or if that file is
outdated.
Parameters:
ebnf_filename(str): The filename of the ebnf-source of the
grammar. In case this is a directory and not a file, all
files within this directory ending with .ebnf will be
compiled.
force(bool): If False (default), the grammar will only be
recompiled if it has been changed.
"""
if os.path.isdir(ebnf_filename):
success = True
for entry in os.listdir(ebnf_filename):
if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
success = success and recompile_grammar(entry, force)
return success
base, ext = os.path.splitext(ebnf_filename)
compiler_name = base + 'Compiler.py'
error_file_name = base + '_ebnf_ERRORS.txt'
errors = []
if (not os.path.exists(compiler_name) or force or
grammar_changed(compiler_name, ebnf_filename)):
# print("recompiling parser for: " + ebnf_filename)
errors = compile_on_disk(ebnf_filename)
if errors:
# print("Errors while compiling: " + ebnf_filename + '!')
with open(error_file_name, 'w') as f:
for e in errors:
f.write(e)
f.write('\n')
return False
if not errors and os.path.exists(error_file_name):
os.remove(error_file_name)
return True
......@@ -61,10 +61,10 @@ except ImportError:
import re
try:
from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
try:
from typing import Collection
except ImportError:
pass
# try:
# from typing import Collection
# except ImportError:
# pass
except ImportError:
from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
......@@ -73,7 +73,7 @@ from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, Pa
Node, TransformationFunc
from DHParser.toolkit import load_if_file, error_messages
__all__ = ['PreprocessorFunc',
__all__ = ('PreprocessorFunc',
'HistoryRecord',
'Parser',
'Grammar',
......@@ -109,7 +109,7 @@ __all__ = ['PreprocessorFunc',
'Pop',
'Forward',
'Compiler',
'compile_source']
'compile_source')
......
......@@ -33,14 +33,14 @@ except ImportError:
from DHParser.toolkit import log_dir, line_col
__all__ = ['WHITESPACE_PTYPE',
__all__ = ('WHITESPACE_PTYPE',
'TOKEN_PTYPE',
'ZOMBIE_PARSER',
'ParserBase',
'Error',
'Node',
'TransformationFunc']
'mock_syntax_tree',
'TransformationFunc')
class ParserBase:
......@@ -461,4 +461,56 @@ class Node:
# return nav(path.split('/'))
def mock_syntax_tree(sexpr):
"""
Generates a tree of nodes from an S-expression.
Example:
>>> mock_syntax_tree("(a (b c))").as_sxpr()
'(a\\n (b\\n "c"\\n )\\n)'
"""
def next_block(s):
s = s.strip()
while s[0] != ')':
if s[0] != '(': raise ValueError('"(" expected, not ' + s[:10])
# assert s[0] == '(', s
level = 1
i = 1
while level > 0:
if s[i] == '(':
level += 1
elif s[i] == ')':
level -= 1
i += 1
yield s[:i]
s = s[i:].strip()
sexpr = sexpr.strip()
if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
# assert sexpr[0] == '(', sexpr
sexpr = sexpr[1:].strip()
m = re.match('[\w:]+', sexpr)
name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
sexpr = sexpr[m.end():].strip()
if sexpr[0] == '(':
result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
else:
lines = []
while sexpr and sexpr[0] != ')':
for qm in ['"""', "'''", '"', "'"]:
m = re.match(qm + r'.*?' + qm, sexpr)
if m:
i = len(qm)
lines.append(sexpr[i:m.end() - i])
sexpr = sexpr[m.end():].strip()
break
else:
m = re.match(r'(?:(?!\)).)*', sexpr)
lines.append(sexpr[:m.end()])
sexpr = sexpr[m.end():]
result = "\n".join(lines)
return Node(MockParser(name, ':' + class_name), result)
TransformationFunc = Union[Callable[[Node], Any], partial]
......@@ -26,15 +26,11 @@ try:
except ImportError:
import re
from DHParser import Node, error_messages
from DHParser import error_messages
from DHParser.toolkit import compact_sexpr, is_logging
from DHParser.syntaxtree import MockParser
from DHParser.ebnf import grammar_changed
from DHParser.dsl import compile_on_disk
from DHParser.syntaxtree import mock_syntax_tree
__all__ = ('mock_syntax_tree',
'recompile_grammar',
'unit_from_configfile',
__all__ = ('unit_from_configfile',
'unit_from_json',
'unit_from_file',
'get_report',
......@@ -42,100 +38,6 @@ __all__ = ('mock_syntax_tree',
'grammar_suite',
'runner')
def mock_syntax_tree(sexpr):
"""
Generates a tree of nodes from an S-expression.
Example:
>>> mock_syntax_tree("(a (b c))").as_sxpr()
'(a\\n (b\\n "c"\\n )\\n)'
"""
def next_block(s):
s = s.strip()
while s[0] != ')':
if s[0] != '(': raise ValueError('"(" expected, not ' + s[:10])
# assert s[0] == '(', s
level = 1
i = 1
while level > 0:
if s[i] == '(':
level += 1
elif s[i] == ')':
level -= 1
i += 1
yield s[:i]
s = s[i:].strip()
sexpr = sexpr.strip()
if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
# assert sexpr[0] == '(', sexpr
sexpr = sexpr[1:].strip()
m = re.match('[\w:]+', sexpr)
name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
sexpr = sexpr[m.end():].strip()
if sexpr[0] == '(':
result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
else:
lines = []
while sexpr and sexpr[0] != ')':
for qm in ['"""', "'''", '"', "'"]:
m = re.match(qm + r'.*?' + qm, sexpr)
if m:
i = len(qm)
lines.append(sexpr[i:m.end() - i])
sexpr = sexpr[m.end():].strip()
break
else:
m = re.match(r'(?:(?!\)).)*', sexpr)
lines.append(sexpr[:m.end()])
sexpr = sexpr[m.end():]
result = "\n".join(lines)
return Node(MockParser(name, ':' + class_name), result)
def recompile_grammar(ebnf_filename, force=False) -> bool:
"""
Recompiles an ebnf-grammar if necessary, that is if either no
corresponding 'XXXXCompiler.py'-file exists or if that file is
outdated.
Parameters:
ebnf_filename(str): The filename of the ebnf-source of the
grammar. In case this is a directory and not a file all
files within this directory ending with .ebnf will be
compiled.
force(bool): If False (default), the grammar will only be
recompiled if it has been changed.
"""
if os.path.isdir(ebnf_filename):
success = True
for entry in os.listdir(ebnf_filename):
if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
success = success and recompile_grammar(entry, force)
return success
base, ext = os.path.splitext(ebnf_filename)
compiler_name = base + 'Compiler.py'
error_file_name = base + '_ebnf_ERRORS.txt'
errors = []
if (not os.path.exists(compiler_name) or force or
grammar_changed(compiler_name, ebnf_filename)):
# print("recompiling parser for: " + ebnf_filename)
errors = compile_on_disk(ebnf_filename)
if errors:
# print("Errors while compiling: " + ebnf_filename + '!')
with open(error_file_name, 'w') as f:
for e in errors:
f.write(e)
f.write('\n')
return False
if not errors and os.path.exists(error_file_name):
os.remove(error_file_name)
return True
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
......
......@@ -45,8 +45,7 @@ try:
except ImportError:
from .typing34 import Any, List, Tuple
__all__ = ['logging',
__all__ = ('logging',
'is_logging',
'log_dir',
'logfile_basename',
......@@ -63,7 +62,7 @@ __all__ = ['logging',
'md5',
'expand_table',
'smart_list',
'sane_parser_name']
'sane_parser_name')
def log_dir() -> str:
......
......@@ -52,6 +52,7 @@ __all__ = ('transformation_factory',
'is_token',
'has_name',
'has_content',
'remove_children_if',
'remove_parser',
'remove_content',
'remove_first',
......@@ -61,6 +62,7 @@ __all__ = ('transformation_factory',
'remove_expendables',
'remove_brackets',
'remove_tokens',
'keep_children',
'flatten',
'forbid',
'require',
......
......@@ -16,5 +16,5 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
__all__ = ('__version__')
__version__ = '0.7.6' # + '_dev' + str(os.stat(__file__).st_mtime)
__all__ = ['__version__']
......@@ -3,3 +3,5 @@ Folder "DevScripts"
This folder contains helper scripts for the
development of DHParser.
collect_symbols.py - Lists all exported symbols from DHParser modules
"""collect_symbols.py - EBNF -> Lists all exported symbols from DHParser
"""collect_symbols.py - Lists all exported symbols from DHParser modules
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
......
"""create_standalone.py - merges the DHParser modules into a standalone
DHParser.py module for easier deployment.
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import functools
import operator
import sys
sys.path.append('../')
try:
import regex as re
except ImportError:
import re
modules = ('toolkit', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber')
all_symbols = list(functools.reduce(operator.or_, (set(eval(m + '.__all__')) for m in modules)))
all_symbols.sort()
def start(module):
i = module.find('__all__')
i = module.find(')', i)
i = module.find('\n', i) + 1
return i
doc = "DHParser.py - Packrat-parser and parser-generator\n\n" + __doc__[__doc__.find('Copyright'):]
imports = """
import abc
import codecs
import collections
from collections import OrderedDict
import configparser
import contextlib
import copy
from functools import partial
import hashlib
import inspect
import json
import os
import platform
try:
import regex as re
except ImportError:
import re
import sys
from typing import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \\
Iterator, List, NamedTuple, Sequence, Set, Union, Text, Tuple
"""
heading = """
#######################################################################
#######################################################################
#
# %s
#
#######################################################################
#######################################################################
"""
def merge_modules(module_names, dhp_path='../DHParser/'):
components = [doc, imports]
for name in module_names:
with open(dhp_path + '%s.py' % name) as f:
module = f.read()
content = module[start(module):]
components.append(heading % name)
components.append(content)
return "\n".join(components)
print(merge_modules(modules))
......@@ -20,14 +20,13 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import cProfile
import functools
import sys
sys.path.extend(['../../', '../', './'])
from DHParser.toolkit import logging
from DHParser.testing import recompile_grammar
from DHParser.dsl import recompile_grammar
def profile(func):
......
......@@ -20,10 +20,14 @@ limitations under the License.
"""
import sys
import DHParser.dsl
sys.path.extend(['../../', '../', './'])
from DHParser import testing
if not testing.recompile_grammar('LaTeX.ebnf', force=True): # recompiles Grammar only if it has changed
if not DHParser.dsl.recompile_grammar('LaTeX.ebnf', force=True): # recompiles Grammar only if it has changed
with open('LaTeX_ebnf_ERRORS.txt') as f:
print(f.read())
sys.exit(1)
......
......@@ -20,7 +20,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
from DHParser.testing import recompile_grammar
from DHParser.dsl import recompile_grammar
recompile_grammar('.')
# import os
......
......@@ -26,7 +26,7 @@ import sys
sys.path.extend(['../../', '../', './'])
flag = os.path.exists('LyrikCompiler.py')
from DHParser.testing import recompile_grammar
from DHParser.dsl import recompile_grammar
if not recompile_grammar('.', force=True):
with open('Lyrik_ebnf_ERRORS.txt') as f:
......
......@@ -23,10 +23,9 @@ import copy
import sys
sys.path.extend(['../', './'])
from DHParser.syntaxtree import Node, TOKEN_PTYPE
from DHParser.syntaxtree import Node, TOKEN_PTYPE, mock_syntax_tree
from DHParser.transform import traverse, reduce_single_child, \
replace_by_single_child, flatten, remove_expendables
from DHParser.testing import mock_syntax_tree
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import parser_factory
......
......@@ -25,11 +25,11 @@ from functools import partial
sys.path.extend(['../', './'])
from DHParser.toolkit import compact_sexpr
from DHParser.syntaxtree import TOKEN_PTYPE
from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree
from DHParser.transform import traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten
from DHParser.dsl import parser_factory
from DHParser.testing import grammar_unit, mock_syntax_tree
from DHParser.testing import grammar_unit
ARITHMETIC_EBNF = """
@ whitespace = linefeed
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment