Dokumentation ergänzt

.gitignore

@@ -38,3 +38,7 @@ DHParser/stringview.c
 imperium.html
 fascitergula.html
 _build
+examples/Tutorial/LyrikCompiler.py
+_build
+_static
+_templates

DHParser/dsl.py

@@ -23,6 +23,8 @@ compilation of domain specific languages based on an EBNF-grammar.
 
 
 import os
+import platform
+import stat
 
 from DHParser.compile import Compiler, compile_source
 from DHParser.ebnf import EBNFCompiler, grammar_changed, \
@@ -70,10 +72,16 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
 END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
 
 
+dhparserdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+
 DHPARSER_IMPORTS = '''
 from functools import partial
 import os
 import sys
+
+sys.path.append('{dhparserdir}')
+
 try:
     import regex as re
 except ImportError:
@@ -92,7 +100,7 @@ from DHParser import logging, is_filename, load_if_file, \\
     remove_nodes, remove_content, remove_brackets, replace_parser, \\
     keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \\
     remove_anonymous_empty, keep_nodes, traverse_locally, strip
-'''
+'''.format(dhparserdir=dhparserdir)
 
 
 DHPARSER_MAIN = '''
@@ -480,8 +488,9 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
         if RX_WHITESPACE.fullmatch(compiler):
             compiler = ebnf_compiler.gen_compiler_skeleton()
 
+        compilerscript = rootname + 'Compiler.py'
         try:
-            f = open(rootname + 'Compiler.py', 'w', encoding="utf-8")
+            f = open(compilerscript, 'w', encoding="utf-8")
             f.write(intro)
             f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION))
             f.write(imports)
@@ -496,13 +505,18 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
             f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER))
             f.write(outro)
         except (PermissionError, FileNotFoundError, IOError) as error:
-            print('# Could not write file "' + rootname + 'Compiler.py" because of: '
+            print('# Could not write file "' + compilerscript + '" because of: '
                   + "\n# ".join(str(error).split('\n)')))
             print(result)
         finally:
             if f:
                 f.close()
 
+        if platform.system() != "Windows":
+            # set file permissions so that the compilerscript can be executed
+            st = os.stat(compilerscript)
+            os.chmod(compilerscript, st.st_mode | stat.S_IEXEC)
+
     else:
         f = None
         try:

DevScripts/Readme-DevScripts.md

@@ -2,7 +2,7 @@ Folder "DevScripts"
 ===================
 
 This folder contains helper scripts for the development of DHParser.
-**These scripts are experimental and my be out of date!**
+**These scripts are experimental and horribly outdated!**
 
 * collect_symbols.py - Lists all exported symbols from DHParser modules
 

experimental/test_stringview_performance.py → DevScripts/test_stringview_performance.py

 #!/usr/bin/python
 
+import sys
+
+sys.path.append('../')
+
 from DHParser.stringview import StringView
 from timeit import timeit
 import re

Introduction.md

 Introduction to [DHParser](https://gitlab.lrz.de/badw-it/DHParser)
 ==================================================================
 
-*This is just an appetizer. Full documentation coming soon...*
-
 Motto: **Computers enjoy XML, humans don't.**
 
 Why use domain specific languages in the humanities
@@ -160,19 +158,19 @@ The output will be something like this:
 
 Now, you might notice that this is not exactly the XML-encoding as shown
 above. (Can you spot the differences?) But you will probably believe me
-without further proof that it can easily be converted into the other
-version and contains all the information that the other version contains.
+without further proof that it can easily be converted into the other version
+and contains all the information that the other version contains.
 
 How does DHParser achieve this? Well, there is the rub. In order to convert
 the poem in the domain specific version into the XML-version, DHParser
-requires a structural description of the domain specific encoding. This
-is a bit similar to a document type definition (DTD) in XML. This
-structural description uses a slightly enhanced version of the
-[Extended-Backus-Naur-Form (EBNF)](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form),
-which is a well-established formalism for the structural description of
-formal languages in computer sciences. An excerpt of the EBNF-definition
-of our domain-specific encoding for the poem looks like this. (We leave out
-the meta-data here. See
+requires a structural description of the domain specific encoding. This is a
+bit similar to a document type definition (DTD) in XML. This structural
+description uses a slightly enhanced version of the [Extended-Backus-Naur-Form
+(EBNF)](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form),
+which is a well-established formalism for the structural description of formal
+languages in computer sciences. An excerpt of the EBNF-definition of our
+domain-specific encoding for the poem looks like this. (We leave out the
+meta-data here. See
 [`examples/Tutorial/Lyrik.ebnf`](https://gitlab.lrz.de/badw-it/DHParser/blob/master/examples/Tutorial/Lyrik.ebnf)
 for the full EBNF):
 
@@ -192,48 +190,53 @@ for the full EBNF):
 
 Without going into too much detail here, let me just explain a few basics of
 this formal description: The slashes `/` enclose ordinary regular expressions.
-Thus, `NZ` for ("Neue Zeile", German for: "new line") is defined as `/\n/~` which
-is the newline-token `\n` in a regular expression, plus further horizontal
-whitespace (signified by the tilde `~`), if there is any.
-
-The braces `{` `}` enclose items that can be repeated zero or more times; with a
-`+` appended to the closing brace it means one or more times. Now, look at the
-definition of `text` in the 6th line: `{ strophe {LEERZEILE} }+`. This reads as
-follows: The text of the poem consists of a sequence of stanzas, each of which
-is followed by a sequence of empty lines (German: "Leerzeilen"). If you now look
-at the structural definition of a stanza, you find that it consists of a sequence
-of verses, each of which starts, i.e. is preceded by a new line.
-
-Can you figure out the rest? Hint: The angular brackets `[` and `]` mean that and
-item is optional and the `§` sign means that it is obligatory. (Strictly speaking,
-the §-signs are not necessary, because an item that is not optional is always
-obligatory, but the §-signs help the converter to produce more useful error
-messages.)
+Thus, `NZ` for ("Neue Zeile", German for: "new line") is defined as `/\n/~`
+which is the newline-token `\n` in a regular expression, plus further
+horizontal whitespace (signified by the tilde `~`), if there is any.
+
+The braces `{` `}` enclose items that can be repeated zero or more times; with
+a `+` appended to the closing brace it means one or more times. Now, look at
+the definition of `text` in the 6th line: `{ strophe {LEERZEILE} }+`. This
+reads as follows: The text of the poem consists of a sequence of stanzas, each
+of which is followed by a sequence of empty lines (German: "Leerzeilen"). If
+you now look at the structural definition of a stanza, you find that it
+consists of a sequence of verses, each of which starts, i.e. is preceded by a
+new line.
+
+Can you figure out the rest? Hint: The angular brackets `[` and `]` mean that
+and item is optional and the `§` sign means that it is obligatory. (Strictly
+speaking, the §-signs are not necessary, because an item that is not optional
+is always obligatory, but the §-signs help the converter to produce more
+useful error messages.)
 
 This should be enough for an introduction to the purpose of DSLs in the
-humanities. It has shown the probably most important use case of
-DHParser, i.e. as a frontend-technology form XML-encodings. Of course,
-it can just as well be used as a frontend for any other kind of
-structured data, like SQL or graph-structured data. The latter is by the
-way is a very reasonable alternative to XML for edition projects with a
-complex transmission history. See Andreas Kuczera's Blog-entry on
-["Graphdatenbanken für Historiker"](http://mittelalter.hypotheses.org/5995).
-
+humanities. It has shown the probably most important use case of DHParser,
+i.e. as a frontend-technology form XML-encodings. Of course, it can just as
+well be used as a frontend for any other kind of structured data, like SQL or
+graph-structured data. The latter is by the way is a very reasonable
+alternative to XML for edition projects with a complex transmission history.
+See Andreas Kuczera's Blog-entry on ["Graphdatenbanken für
+Historiker"](http://mittelalter.hypotheses.org/5995).
 
 Tutorial: First Steps with DHParser
 -----------------------------------
 
-Disclaimer: *You'll need to be able to use a shell and have some basic
-knowledge of Python programming to be able to follow this section!*
+*You'll need to be able to use a shell and have some basic knowledge of Python
+programming to be able to follow this section!* Also, you need to have
+[git](https://git-scm.com/) and [python 3](https://www.python.org/) installed
+on you system. It is important that you have at least python version 3.5.
+DHParser will not work with python 2. You can simply start python to find out
+which version you have got.
 
 In order to try the example above, you should fetch DHParsers from its
-git-repository:
+git-repository. Open a shell and type:
 
     $ git clone git@gitlab.lrz.de:badw-it/DHParser.git
 
-Now, if you enter the repo, you'll find three subdirectories:
+Now, if you enter the repo, you'll find among others these subdirectories:
 
     DHParser
+    documentation
     examples
     test
 
@@ -266,7 +269,7 @@ generated file that contains the actual parser. All other parts - we
 will come to that later what these are - can safely be edited by you.
 Now just run `recompile_grammar.py` from the command line:
 
-    $ python3 recompile_grammar.py
+    $ python recompile_grammar.py
 
 You'll find that `recompile_grammar.py` has generated a new script with
 the name `LyrikCompiler.py`. This script contains the Parser for the
@@ -274,7 +277,7 @@ the name `LyrikCompiler.py`. This script contains the Parser for the
 rather, a DSL-whatever compiler), which you can later fill in. Now let's
 see how this script works:
 
-    $ python3 LyrikCompiler.py Lyrisches_Intermezzo_IV.txt >result.xml
+    $ python LyrikCompiler.py Lyrisches_Intermezzo_IV.txt >result.xml
 
 The file `Lyrisches_Intermezzo_IV.txt` contains the fourth part of
 Heinrich Heine's Lyrisches Intermezzo encoded in our own human-readable
@@ -316,7 +319,6 @@ recognizable!) first verse of the poem:
     </vers>
     ...
 
-
 How come it is so obfuscated, and where do all those pseudo-tags like
 `<:RegExp>` and `<:Whitespace>` come from? Well, this is probably the
 right time to explain a bit about parsing and compilation in general.
@@ -390,8 +392,7 @@ keeps the specification of the AST-transformation simple and concise. At
 the same, we avoid adding hints for the AST-transformation in the
 grammar specification, which would render the grammar less readable.
 
-Next, I am going to explain step by step, how a domain specific language
-for poems like Heine's Lyrisches Intermezzo can be designed, specified,
-compiled and tested.
-
-*to be continued, stay tuned...*
+Now that you have seen how DHParser basically works, it is time to go
+through the process of desining and testing a domain specific notation
+step by step from the very start. Head over to the documentation in
+subdirectory and read the step by step guide.
\ No newline at end of file

dhparser.py

@@ -29,6 +29,8 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi
 from DHParser.log import logging
 from DHParser.toolkit import re
 
+dhparserdir = os.path.dirname(os.path.realpath(__file__))
+
 EBNF_TEMPLATE = r"""-grammar
 
 #######################################################################
@@ -114,10 +116,11 @@ GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3
 import os
 import sys
 
-# sys.path.extend(['../../', '../', './'])  # use for developing DHParser
+sys.path.append('{dhparserdir}')
 
 scriptpath = os.path.dirname(__file__)
 
+
 try:
     from DHParser import dsl
     import DHParser.log
@@ -141,10 +144,9 @@ def recompile_grammar(grammar_src, force):
 
 def run_grammar_tests(glob_pattern):
     with DHParser.log.logging(False):
-        print(glob_pattern)
         error_report = testing.grammar_suite(
-            os.path.join(scriptpath, 'grammar_tests'), 
-            get_grammar, get_transformer, 
+            os.path.join(scriptpath, 'grammar_tests'),
+            get_grammar, get_transformer,
             fn_patterns=[glob_pattern], report=True, verbose=True)
     return error_report
 
@@ -154,7 +156,7 @@ if __name__ == '__main__':
     if arg.endswith('.ebnf'):
         recompile_grammar(arg, force=True)
     else:
-        recompile_grammar(os.path.join(scriptpath, '{name}.ebnf'), 
+        recompile_grammar(os.path.join(scriptpath, '{name}.ebnf'),
                           force=False)
         sys.path.append('.')
         from {name}Compiler import get_grammar, get_transformer
@@ -180,7 +182,7 @@ def create_project(path: str):
             print('"%s" already exists! Not overwritten.' % name)
 
     name = os.path.basename(path)
-    if not re.match('(?!\d)\w+', name):
+    if not re.match(r'(?!\d)\w+', name):
         print('Project name "%s" is not a valid identifier! Aborting.' % name)
         sys.exit(1)
     if os.path.exists(path) and not os.path.isdir(path):
@@ -202,7 +204,8 @@ def create_project(path: str):
     create_file(os.path.join('grammar_tests', '02_test_document.ini'), TEST_DOCUMENT_TEMPLATE)
     create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE)
     create_file('README.md', README_TEMPLATE.format(name=name))
-    create_file('tst_%s_grammar.py' % name, GRAMMAR_TEST_TEMPLATE.format(name=name))
+    create_file('tst_%s_grammar.py' % name,
+                GRAMMAR_TEST_TEMPLATE.format(name=name, dhparserdir=dhparserdir))
     create_file('example.dsl', 'Life is but a walking shadow\n')
     os.chmod('tst_%s_grammar.py' % name, 0o755)
     os.chdir(curr_dir)

documentation/.buildinfo

+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: a864fbe9973eb04d25fc99fca3d8ce3e
+tags: 645f666f9bcd5a90fca523b33c5a78b7

documentation/.gitkeep

+

documentation/StepByStepGuide.html

+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>DHParser’s Step by Step Guide &mdash; DHParser 0.8 documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="genindex.html"/>
+        <link rel="search" title="Search" href="search.html"/>
+    <link rel="top" title="DHParser 0.8 documentation" href="index.html"/> 
+
+  
+  <script src="_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="index.html" class="icon icon-home"> DHParser
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <p class="caption"><span class="caption-text">Contents:</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="UserGuide.html">DHParser User’s Guide</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ReferenceManual.html">DHParser Reference Manual</a></li>
+<li class="toctree-l1"><a class="reference internal" href="ModuleReference.html">Module Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="index.html">DHParser</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="index.html">Docs</a> &raquo;</li>
+        
+      <li>DHParser’s Step by Step Guide</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+            <a href="_sources/StepByStepGuide.rst.txt" rel="nofollow"> View page source</a>
+          
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <div class="section" id="dhparser-s-step-by-step-guide">
+<h1>DHParser’s Step by Step Guide<a class="headerlink" href="#dhparser-s-step-by-step-guide" title="Permalink to this headline">¶</a></h1>
+<p>This step by step guide goes through the whole process of desining and testing
+a domain specific notation from the very start. (The terms “domain specific
+noation” and “domain specific language” are used interchangeably in the
+following. Both will abbreviated by “DSL”, however.) We will design a simple
+domain specific notation for poems as a teaching example. On the way we will
+learn:</p>
+<ol class="arabic simple">
+<li>how to setup a new DHParser project</li>
+<li>how to use the test driven development approach to designing a DSL</li>
+<li>how to describe the syntax of a DSL with the EBNF-notation</li>
+<li>how to specify the transformations for converting the concrete syntax tree
+that results from parsing a text denoted in our DLS into an abstract syntax
+tree that represents or comes close to representing out data model.</li>
+<li>how to write a compiler that transforms the abstract syntax tree into a
+target representation which might be a html page, epub or printable pdf in
+the case of typical Digital-Humanities-projects.</li>
+</ol>
+<div class="section" id="setting-up-a-new-dhparser-project">
+<h2>Setting up a new DHParser project<a class="headerlink" href="#setting-up-a-new-dhparser-project" title="Permalink to this headline">¶</a></h2>
+<p>Since DHParser, while quite mature in terms of implemented features, is still
+in a pre-first-release state, it is for the time being more recommendable to
+clone the most current version of DHParser from the git-repository rather than
+installing the packages from the Python Package Index (PyPI).</p>
+<p>This section takes you from cloning the DHParser git repository to setting up
+a new DHParser-project in the <code class="docutils literal notranslate"><span class="pre">experimental</span></code>-subdirectory and testing
+whether the setup works. Similarily to current web development practices, most
+of the work with DHParser is done from the shell. In the following, we assume a Unix (Linux) environment. The same can most likely be done on other operating systems in a very similar way, but there might be subtle differences.</p>
+<div class="section" id="installing-dhparser-from-the-git-repository">
+<h3>Installing DHParser from the git repository<a class="headerlink" href="#installing-dhparser-from-the-git-repository" title="Permalink to this headline">¶</a></h3>
+<p>In order to install DHParser from the git repository, open up a shell window
+and type:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>$ git clone git@gitlab.lrz.de:badw-it/DHParser.git
+$ cd DHParser
+</pre></div>
+</div>
+<p>The second command changes to the DHParser directory. Within this directory
+you should recognise the following subdirectories and files. There are more
+files and directories for sure, but those will not concern us for now:</p>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>DHParser/            - the DHParser python packages
+documentation/       - DHParser&#39;s documentation in html-form
+documentation_source - DHParser&#39;s documentation in reStructedText-Format
+examples/            - some exmamples for DHParser (mostly incomplete)
+experimental/        - an empty directory for experimenting
+test/                - DHParser&#39;s unit-tests
+dhparser.py          - DHParser&#39;s command line tool for setting up projects
+README.md            - General information about DHParser
+LICENSE.txt          - DHParser&#39;s license. It&#39;s open source (hooray!)
+Introduction.md      - An introduction and appetizer for DHParser
+</pre></div>
+</div>
+<p>In order to setup a new DHParser project, you run the <code class="docutils literal notranslate"><span class="pre">dhparser.py</span></code>-script
+with the name of the new project. For the sake of the example, let’s type:</p>
+<blockquote>
+<div><p>$ python dhparser.py experimental/poetry</p>
+<p>Creating new DHParser-project “poetry”.
+Creating file “grammar_tests/01_test_word.ini”.
+Creating file “grammar_tests/02_test_document.ini”.
+Creating file “poetry.ebnf”.
+Creating file “README.md”.
+Creating file “tst_poetry_grammar.py”.
+Creating file “example.dsl”.
+ready.</p>
+<p>$ cd experimental/poetry</p>
+</div></blockquote>
+<p>This creates a new DHParser-project with the name “poetry” in directory with the same name within the subdirectory “experimental”.</p>
+</div>
+</div>
+</div>
+
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright 2018, Eckhart Arnold.
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'./',
+            VERSION:'0.8',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="_static/jquery.js"></script>
+      <script type="text/javascript" src="_static/underscore.js"></script>
+      <script type="text/javascript" src="_static/doctools.js"></script>
+      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

documentation/UserGuide.html

+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>DHParser User’s Guide &mdash; DHParser 0.8 documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+