Commit c1bd21ce authored by eckhart's avatar eckhart
Browse files

Dokumentation ergänzt

parent 16840e3a
...@@ -38,3 +38,7 @@ DHParser/stringview.c ...@@ -38,3 +38,7 @@ DHParser/stringview.c
imperium.html imperium.html
fascitergula.html fascitergula.html
_build _build
examples/Tutorial/LyrikCompiler.py
_build
_static
_templates
No preview for this file type
...@@ -23,6 +23,8 @@ compilation of domain specific languages based on an EBNF-grammar. ...@@ -23,6 +23,8 @@ compilation of domain specific languages based on an EBNF-grammar.
import os import os
import platform
import stat
from DHParser.compile import Compiler, compile_source from DHParser.compile import Compiler, compile_source
from DHParser.ebnf import EBNFCompiler, grammar_changed, \ from DHParser.ebnf import EBNFCompiler, grammar_changed, \
...@@ -70,10 +72,16 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved. ...@@ -70,10 +72,16 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS" END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
dhparserdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
DHPARSER_IMPORTS = ''' DHPARSER_IMPORTS = '''
from functools import partial from functools import partial
import os import os
import sys import sys
sys.path.append('{dhparserdir}')
try: try:
import regex as re import regex as re
except ImportError: except ImportError:
...@@ -92,7 +100,7 @@ from DHParser import logging, is_filename, load_if_file, \\ ...@@ -92,7 +100,7 @@ from DHParser import logging, is_filename, load_if_file, \\
remove_nodes, remove_content, remove_brackets, replace_parser, \\ remove_nodes, remove_content, remove_brackets, replace_parser, \\
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \\ keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \\
remove_anonymous_empty, keep_nodes, traverse_locally, strip remove_anonymous_empty, keep_nodes, traverse_locally, strip
''' '''.format(dhparserdir=dhparserdir)
DHPARSER_MAIN = ''' DHPARSER_MAIN = '''
...@@ -480,8 +488,9 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -480,8 +488,9 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
if RX_WHITESPACE.fullmatch(compiler): if RX_WHITESPACE.fullmatch(compiler):
compiler = ebnf_compiler.gen_compiler_skeleton() compiler = ebnf_compiler.gen_compiler_skeleton()
compilerscript = rootname + 'Compiler.py'
try: try:
f = open(rootname + 'Compiler.py', 'w', encoding="utf-8") f = open(compilerscript, 'w', encoding="utf-8")
f.write(intro) f.write(intro)
f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION)) f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION))
f.write(imports) f.write(imports)
...@@ -496,13 +505,18 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -496,13 +505,18 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER)) f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER))
f.write(outro) f.write(outro)
except (PermissionError, FileNotFoundError, IOError) as error: except (PermissionError, FileNotFoundError, IOError) as error:
print('# Could not write file "' + rootname + 'Compiler.py" because of: ' print('# Could not write file "' + compilerscript + '" because of: '
+ "\n# ".join(str(error).split('\n)'))) + "\n# ".join(str(error).split('\n)')))
print(result) print(result)
finally: finally:
if f: if f:
f.close() f.close()
if platform.system() != "Windows":
# set file permissions so that the compilerscript can be executed
st = os.stat(compilerscript)
os.chmod(compilerscript, st.st_mode | stat.S_IEXEC)
else: else:
f = None f = None
try: try:
......
...@@ -2,7 +2,7 @@ Folder "DevScripts" ...@@ -2,7 +2,7 @@ Folder "DevScripts"
=================== ===================
This folder contains helper scripts for the development of DHParser. This folder contains helper scripts for the development of DHParser.
**These scripts are experimental and my be out of date!** **These scripts are experimental and horribly outdated!**
* collect_symbols.py - Lists all exported symbols from DHParser modules * collect_symbols.py - Lists all exported symbols from DHParser modules
......
#!/usr/bin/python #!/usr/bin/python
import sys
sys.path.append('../')
from DHParser.stringview import StringView from DHParser.stringview import StringView
from timeit import timeit from timeit import timeit
import re import re
......
Introduction to [DHParser](https://gitlab.lrz.de/badw-it/DHParser) Introduction to [DHParser](https://gitlab.lrz.de/badw-it/DHParser)
================================================================== ==================================================================
*This is just an appetizer. Full documentation coming soon...*
Motto: **Computers enjoy XML, humans don't.** Motto: **Computers enjoy XML, humans don't.**
Why use domain specific languages in the humanities Why use domain specific languages in the humanities
...@@ -160,19 +158,19 @@ The output will be something like this: ...@@ -160,19 +158,19 @@ The output will be something like this:
Now, you might notice that this is not exactly the XML-encoding as shown Now, you might notice that this is not exactly the XML-encoding as shown
above. (Can you spot the differences?) But you will probably believe me above. (Can you spot the differences?) But you will probably believe me
without further proof that it can easily be converted into the other without further proof that it can easily be converted into the other version
version and contains all the information that the other version contains. and contains all the information that the other version contains.
How does DHParser achieve this? Well, there is the rub. In order to convert How does DHParser achieve this? Well, there is the rub. In order to convert
the poem in the domain specific version into the XML-version, DHParser the poem in the domain specific version into the XML-version, DHParser
requires a structural description of the domain specific encoding. This requires a structural description of the domain specific encoding. This is a
is a bit similar to a document type definition (DTD) in XML. This bit similar to a document type definition (DTD) in XML. This structural
structural description uses a slightly enhanced version of the description uses a slightly enhanced version of the [Extended-Backus-Naur-Form
[Extended-Backus-Naur-Form (EBNF)](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), (EBNF)](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form),
which is a well-established formalism for the structural description of which is a well-established formalism for the structural description of formal
formal languages in computer sciences. An excerpt of the EBNF-definition languages in computer sciences. An excerpt of the EBNF-definition of our
of our domain-specific encoding for the poem looks like this. (We leave out domain-specific encoding for the poem looks like this. (We leave out the
the meta-data here. See meta-data here. See
[`examples/Tutorial/Lyrik.ebnf`](https://gitlab.lrz.de/badw-it/DHParser/blob/master/examples/Tutorial/Lyrik.ebnf) [`examples/Tutorial/Lyrik.ebnf`](https://gitlab.lrz.de/badw-it/DHParser/blob/master/examples/Tutorial/Lyrik.ebnf)
for the full EBNF): for the full EBNF):
...@@ -192,48 +190,53 @@ for the full EBNF): ...@@ -192,48 +190,53 @@ for the full EBNF):
Without going into too much detail here, let me just explain a few basics of Without going into too much detail here, let me just explain a few basics of
this formal description: The slashes `/` enclose ordinary regular expressions. this formal description: The slashes `/` enclose ordinary regular expressions.
Thus, `NZ` for ("Neue Zeile", German for: "new line") is defined as `/\n/~` which Thus, `NZ` for ("Neue Zeile", German for: "new line") is defined as `/\n/~`
is the newline-token `\n` in a regular expression, plus further horizontal which is the newline-token `\n` in a regular expression, plus further
whitespace (signified by the tilde `~`), if there is any. horizontal whitespace (signified by the tilde `~`), if there is any.
The braces `{` `}` enclose items that can be repeated zero or more times; with a The braces `{` `}` enclose items that can be repeated zero or more times; with
`+` appended to the closing brace it means one or more times. Now, look at the a `+` appended to the closing brace it means one or more times. Now, look at
definition of `text` in the 6th line: `{ strophe {LEERZEILE} }+`. This reads as the definition of `text` in the 6th line: `{ strophe {LEERZEILE} }+`. This
follows: The text of the poem consists of a sequence of stanzas, each of which reads as follows: The text of the poem consists of a sequence of stanzas, each
is followed by a sequence of empty lines (German: "Leerzeilen"). If you now look of which is followed by a sequence of empty lines (German: "Leerzeilen"). If
at the structural definition of a stanza, you find that it consists of a sequence you now look at the structural definition of a stanza, you find that it
of verses, each of which starts, i.e. is preceded by a new line. consists of a sequence of verses, each of which starts, i.e. is preceded by a
new line.
Can you figure out the rest? Hint: The angular brackets `[` and `]` mean that and
item is optional and the `§` sign means that it is obligatory. (Strictly speaking, Can you figure out the rest? Hint: The angular brackets `[` and `]` mean that
the §-signs are not necessary, because an item that is not optional is always and item is optional and the `§` sign means that it is obligatory. (Strictly
obligatory, but the §-signs help the converter to produce more useful error speaking, the §-signs are not necessary, because an item that is not optional
messages.) is always obligatory, but the §-signs help the converter to produce more
useful error messages.)
This should be enough for an introduction to the purpose of DSLs in the This should be enough for an introduction to the purpose of DSLs in the
humanities. It has shown the probably most important use case of humanities. It has shown the probably most important use case of DHParser,
DHParser, i.e. as a frontend-technology form XML-encodings. Of course, i.e. as a frontend-technology form XML-encodings. Of course, it can just as
it can just as well be used as a frontend for any other kind of well be used as a frontend for any other kind of structured data, like SQL or
structured data, like SQL or graph-structured data. The latter is by the graph-structured data. The latter is by the way is a very reasonable
way is a very reasonable alternative to XML for edition projects with a alternative to XML for edition projects with a complex transmission history.
complex transmission history. See Andreas Kuczera's Blog-entry on See Andreas Kuczera's Blog-entry on ["Graphdatenbanken für
["Graphdatenbanken für Historiker"](http://mittelalter.hypotheses.org/5995). Historiker"](http://mittelalter.hypotheses.org/5995).
Tutorial: First Steps with DHParser Tutorial: First Steps with DHParser
----------------------------------- -----------------------------------
Disclaimer: *You'll need to be able to use a shell and have some basic *You'll need to be able to use a shell and have some basic knowledge of Python
knowledge of Python programming to be able to follow this section!* programming to be able to follow this section!* Also, you need to have
[git](https://git-scm.com/) and [python 3](https://www.python.org/) installed
on you system. It is important that you have at least python version 3.5.
DHParser will not work with python 2. You can simply start python to find out
which version you have got.
In order to try the example above, you should fetch DHParsers from its In order to try the example above, you should fetch DHParsers from its
git-repository: git-repository. Open a shell and type:
$ git clone git@gitlab.lrz.de:badw-it/DHParser.git $ git clone git@gitlab.lrz.de:badw-it/DHParser.git
Now, if you enter the repo, you'll find three subdirectories: Now, if you enter the repo, you'll find among others these subdirectories:
DHParser DHParser
documentation
examples examples
test test
...@@ -266,7 +269,7 @@ generated file that contains the actual parser. All other parts - we ...@@ -266,7 +269,7 @@ generated file that contains the actual parser. All other parts - we
will come to that later what these are - can safely be edited by you. will come to that later what these are - can safely be edited by you.
Now just run `recompile_grammar.py` from the command line: Now just run `recompile_grammar.py` from the command line:
$ python3 recompile_grammar.py $ python recompile_grammar.py
You'll find that `recompile_grammar.py` has generated a new script with You'll find that `recompile_grammar.py` has generated a new script with
the name `LyrikCompiler.py`. This script contains the Parser for the the name `LyrikCompiler.py`. This script contains the Parser for the
...@@ -274,7 +277,7 @@ the name `LyrikCompiler.py`. This script contains the Parser for the ...@@ -274,7 +277,7 @@ the name `LyrikCompiler.py`. This script contains the Parser for the
rather, a DSL-whatever compiler), which you can later fill in. Now let's rather, a DSL-whatever compiler), which you can later fill in. Now let's
see how this script works: see how this script works:
$ python3 LyrikCompiler.py Lyrisches_Intermezzo_IV.txt >result.xml $ python LyrikCompiler.py Lyrisches_Intermezzo_IV.txt >result.xml
The file `Lyrisches_Intermezzo_IV.txt` contains the fourth part of The file `Lyrisches_Intermezzo_IV.txt` contains the fourth part of
Heinrich Heine's Lyrisches Intermezzo encoded in our own human-readable Heinrich Heine's Lyrisches Intermezzo encoded in our own human-readable
...@@ -316,7 +319,6 @@ recognizable!) first verse of the poem: ...@@ -316,7 +319,6 @@ recognizable!) first verse of the poem:
</vers> </vers>
... ...
How come it is so obfuscated, and where do all those pseudo-tags like How come it is so obfuscated, and where do all those pseudo-tags like
`<:RegExp>` and `<:Whitespace>` come from? Well, this is probably the `<:RegExp>` and `<:Whitespace>` come from? Well, this is probably the
right time to explain a bit about parsing and compilation in general. right time to explain a bit about parsing and compilation in general.
...@@ -390,8 +392,7 @@ keeps the specification of the AST-transformation simple and concise. At ...@@ -390,8 +392,7 @@ keeps the specification of the AST-transformation simple and concise. At
the same, we avoid adding hints for the AST-transformation in the the same, we avoid adding hints for the AST-transformation in the
grammar specification, which would render the grammar less readable. grammar specification, which would render the grammar less readable.
Next, I am going to explain step by step, how a domain specific language Now that you have seen how DHParser basically works, it is time to go
for poems like Heine's Lyrisches Intermezzo can be designed, specified, through the process of desining and testing a domain specific notation
compiled and tested. step by step from the very start. Head over to the documentation in
subdirectory and read the step by step guide.
*to be continued, stay tuned...* \ No newline at end of file
...@@ -29,6 +29,8 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi ...@@ -29,6 +29,8 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi
from DHParser.log import logging from DHParser.log import logging
from DHParser.toolkit import re from DHParser.toolkit import re
dhparserdir = os.path.dirname(os.path.realpath(__file__))
EBNF_TEMPLATE = r"""-grammar EBNF_TEMPLATE = r"""-grammar
####################################################################### #######################################################################
...@@ -114,10 +116,11 @@ GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3 ...@@ -114,10 +116,11 @@ GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3
import os import os
import sys import sys
# sys.path.extend(['../../', '../', './']) # use for developing DHParser sys.path.append('{dhparserdir}')
scriptpath = os.path.dirname(__file__) scriptpath = os.path.dirname(__file__)
try: try:
from DHParser import dsl from DHParser import dsl
import DHParser.log import DHParser.log
...@@ -141,7 +144,6 @@ def recompile_grammar(grammar_src, force): ...@@ -141,7 +144,6 @@ def recompile_grammar(grammar_src, force):
def run_grammar_tests(glob_pattern): def run_grammar_tests(glob_pattern):
with DHParser.log.logging(False): with DHParser.log.logging(False):
print(glob_pattern)
error_report = testing.grammar_suite( error_report = testing.grammar_suite(
os.path.join(scriptpath, 'grammar_tests'), os.path.join(scriptpath, 'grammar_tests'),
get_grammar, get_transformer, get_grammar, get_transformer,
...@@ -180,7 +182,7 @@ def create_project(path: str): ...@@ -180,7 +182,7 @@ def create_project(path: str):
print('"%s" already exists! Not overwritten.' % name) print('"%s" already exists! Not overwritten.' % name)
name = os.path.basename(path) name = os.path.basename(path)
if not re.match('(?!\d)\w+', name): if not re.match(r'(?!\d)\w+', name):
print('Project name "%s" is not a valid identifier! Aborting.' % name) print('Project name "%s" is not a valid identifier! Aborting.' % name)
sys.exit(1) sys.exit(1)
if os.path.exists(path) and not os.path.isdir(path): if os.path.exists(path) and not os.path.isdir(path):
...@@ -202,7 +204,8 @@ def create_project(path: str): ...@@ -202,7 +204,8 @@ def create_project(path: str):
create_file(os.path.join('grammar_tests', '02_test_document.ini'), TEST_DOCUMENT_TEMPLATE) create_file(os.path.join('grammar_tests', '02_test_document.ini'), TEST_DOCUMENT_TEMPLATE)
create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE) create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE)
create_file('README.md', README_TEMPLATE.format(name=name)) create_file('README.md', README_TEMPLATE.format(name=name))
create_file('tst_%s_grammar.py' % name, GRAMMAR_TEST_TEMPLATE.format(name=name)) create_file('tst_%s_grammar.py' % name,
GRAMMAR_TEST_TEMPLATE.format(name=name, dhparserdir=dhparserdir))
create_file('example.dsl', 'Life is but a walking shadow\n') create_file('example.dsl', 'Life is but a walking shadow\n')
os.chmod('tst_%s_grammar.py' % name, 0o755) os.chmod('tst_%s_grammar.py' % name, 0o755)
os.chdir(curr_dir) os.chdir(curr_dir)
......
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: a864fbe9973eb04d25fc99fca3d8ce3e
tags: 645f666f9bcd5a90fca523b33c5a78b7
This source diff could not be displayed because it is too large. You can view the blob instead.
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>DHParser Reference Manual &mdash; DHParser 0.8 documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="DHParser 0.8 documentation" href="index.html"/>
<link rel="next" title="Module Reference" href="ModuleReference.html"/>
<link rel="prev" title="DHParser User’s Guide" href="UserGuide.html"/>
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> DHParser
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="UserGuide.html">DHParser User’s Guide</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">DHParser Reference Manual</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#fundamentals">Fundamentals</a></li>
<li class="toctree-l2"><a class="reference internal" href="#compiler-creation-workflow">Compiler Creation Workflow</a></li>
<li class="toctree-l2"><a class="reference internal" href="#component-guide">Component Guide</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#parser">Parser</a></li>
<li class="toctree-l3"><a class="reference internal" href="#ast-transformation">AST-Transformation</a></li>
<li class="toctree-l3"><a class="reference internal" href="#compiler">Compiler</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#module-structure-of-dhparser">Module Structure of DHParser</a></li>
<li class="toctree-l2"><a class="reference internal" href="#class-hierarchy-of-dhparser">Class Hierarchy of DHParser</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="ModuleReference.html">Module Reference</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">DHParser</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li>DHParser Reference Manual</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/ReferenceManual.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="dhparser-reference-manual">
<h1>DHParser Reference Manual<a class="headerlink" href="#dhparser-reference-manual" title="Permalink to this headline"></a></h1>
<p>This reference manual explains the technology used by DHParser. It is
intended for people who would like to extend or contribute to
DHParser. The reference manual does not explain how a Domain Specific
Language (DSL) is developed (see the User’s Manual for that). It it
explains the technical approach that DHParser employs for parsing,
abstract syntax tree transformation and compilation of a given
DSL. And it describes the module and class structure of the DHParser
Software. The programming guide requires a working knowledge of Python
programming and a basic understanding or common parser technology from
the reader. Also, it is recommended to read the introduction and the
user’s guide first.</p>
<div class="section" id="fundamentals">
<h2>Fundamentals<a class="headerlink" href="#fundamentals" title="Permalink to this headline"></a></h2>
<p>DHParser is a parser generator aimed at but not restricted to the
creation of domain specific languages in the Digital Humanities (DH),
hence the name “DHParser”. In the Digital Humanities, DSLs allow to
enter annotated texts or data in a human friendly and readable form
with a Text-Editor. In contrast to the prevailing XML-approach, the
DSL-approach distinguishes between a human-friendly <em>editing data
format</em> and a maschine friendly <em>working data format</em> which can be XML
but does not need to be. Therefore, the DSL-approach requires an
additional step to reach the <em>working data format</em>, that is, the
compilation of the annotated text or data written in the DSL (editing
data format) to the working data format. In the following a text or
data file wirtten in a DSL will simply be called <em>document</em>. The
editing data format will also be called <em>source format</em> and the
working data format be denoted as <em>target format</em>.</p>
<p>Compiling a document specified in a domain specific language involves the following steps:</p>
<ol class="arabic simple">
<li><strong>Parsing</strong> the document which results in a representation of the document as a concrete
syntax tree.</li>
<li><strong>Transforming</strong> the concrete syntax tree (CST) into an abstract syntax tree (AST), i.e. a
streamlined and simplified syntax tree ready for compilation.</li>
<li><strong>Compiling</strong> the abstract syntax tree into the working data format.</li>
</ol>
<p>All of these steps a carried out be the computer without any user intervention, i.e. without the
need of humans to rewrite or enrich the data during any these steps. A DSL-compiler therefore
consists of three components which are applied in sequence, a <em>parser</em>, a <em>transformer</em> and a
<em>compiler</em>. Creating, i.e. programming these components is the task of compiler construction.
The creation of all of these components is supported by DHParser, albeit to a different degree:</p>