Commit 39979300 authored by eckhart's avatar eckhart

documentation updated

parent 9fc9c2d2
......@@ -144,7 +144,8 @@ class Compiler:
@staticmethod
def method_name(node_name: str) -> str:
"""Returns the method name for `node_name`, e.g.::
"""
Returns the method name for `node_name`, e.g.::
>>> Compiler.method_name('expression')
'on_expression'
......
......@@ -77,52 +77,53 @@ def get_ebnf_preprocessor() -> PreprocessorFunc:
class EBNFGrammar(Grammar):
r"""Parser for an EBNF source file, with this grammar:
# EBNF-Grammar in EBNF
@ comment = /#.*(?:\n|$)/ # comments start with '#' and eat all chars up to and including '\n'
@ whitespace = /\s*/ # whitespace includes linefeed
@ literalws = right # trailing whitespace of literals will be ignored tacitly
syntax = [~//] { definition | directive } §EOF
definition = symbol §"=" expression
directive = "@" §symbol "=" ( regexp | literal | list_ )
expression = term { "|" term }
term = { ["§"] factor }+ # "§" means all following factors mandatory
factor = [flowmarker] [retrieveop] symbol !"=" # negative lookahead to be sure it's not a definition
| [flowmarker] literal
| [flowmarker] plaintext
| [flowmarker] regexp
| [flowmarker] whitespace
| [flowmarker] oneormore
| [flowmarker] group
| [flowmarker] unordered
| repetition
| option
flowmarker = "!" | "&" # '!' negative lookahead, '&' positive lookahead
| "-!" | "-&" # '-' negative lookbehind, '-&' positive lookbehind
retrieveop = "::" | ":" # '::' pop, ':' retrieve
group = "(" §expression ")"
unordered = "<" §expression ">" # elements of expression in arbitrary order
oneormore = "{" expression "}+"
repetition = "{" §expression "}"
option = "[" §expression "]"
symbol = /(?!\d)\w+/~ # e.g. expression, factor, parameter_list
literal = /"(?:[^"]|\\")*?"/~ # e.g. "(", '+', 'while'
| /'(?:[^']|\\')*?'/~ # whitespace following literals will be ignored tacitly.
plaintext = /`(?:[^"]|\\")*?`/~ # like literal but does not eat whitespace
regexp = /~?\/(?:\\\/|[^\/])*?\/~?/~ # e.g. /\w+/, ~/#.*(?:\n|$)/~
# '~' is a whitespace-marker, if present leading or trailing
# whitespace of a regular expression will be ignored tacitly.
whitespace = /~/~ # implicit or default whitespace
list_ = /\w+/~ { "," /\w+/~ } # comma separated list of symbols, e.g. BEGIN_LIST, END_LIST,
# BEGIN_QUOTE, END_QUOTE ; see CommonMark/markdown.py for an exmaple
EOF = !/./
r"""
Parser for an EBNF source file, with this grammar::
# EBNF-Grammar in EBNF
@ comment = /#.*(?:\n|$)/ # comments start with '#' and eat all chars up to and including '\n'
@ whitespace = /\s*/ # whitespace includes linefeed
@ literalws = right # trailing whitespace of literals will be ignored tacitly
syntax = [~//] { definition | directive } §EOF
definition = symbol §"=" expression
directive = "@" §symbol "=" ( regexp | literal | list_ )
expression = term { "|" term }
term = { ["§"] factor }+ # "§" means all following factors mandatory
factor = [flowmarker] [retrieveop] symbol !"=" # negative lookahead to be sure it's not a definition
| [flowmarker] literal
| [flowmarker] plaintext
| [flowmarker] regexp
| [flowmarker] whitespace
| [flowmarker] oneormore
| [flowmarker] group
| [flowmarker] unordered
| repetition
| option
flowmarker = "!" | "&" # '!' negative lookahead, '&' positive lookahead
| "-!" | "-&" # '-' negative lookbehind, '-&' positive lookbehind
retrieveop = "::" | ":" # '::' pop, ':' retrieve
group = "(" §expression ")"
unordered = "<" §expression ">" # elements of expression in arbitrary order
oneormore = "{" expression "}+"
repetition = "{" §expression "}"
option = "[" §expression "]"
symbol = /(?!\d)\w+/~ # e.g. expression, factor, parameter_list
literal = /"(?:[^"]|\\")*?"/~ # e.g. "(", '+', 'while'
| /'(?:[^']|\\')*?'/~ # whitespace following literals will be ignored tacitly.
plaintext = /`(?:[^"]|\\")*?`/~ # like literal but does not eat whitespace
regexp = /~?\/(?:\\\/|[^\/])*?\/~?/~ # e.g. /\w+/, ~/#.*(?:\n|$)/~
# '~' is a whitespace-marker, if present leading or trailing
# whitespace of a regular expression will be ignored tacitly.
whitespace = /~/~ # implicit or default whitespace
list_ = /\w+/~ { "," /\w+/~ } # comma separated list of symbols, e.g. BEGIN_LIST, END_LIST,
# BEGIN_QUOTE, END_QUOTE ; see CommonMark/markdown.py for an exmaple
EOF = !/./
"""
expression = Forward()
source_hash__ = "3fc9f5a340f560e847d9af0b61a68743"
......
......@@ -354,7 +354,7 @@ class Grammar:
("direct instantiation"), or by assigning the root parser to the
class variable "root__" of a descendant class of class Grammar.
Example for direct instantiation of a grammar:
Example for direct instantiation of a grammar::
>>> number = RE('\d+') + RE('\.') + RE('\d+') | RE('\d+')
>>> number_parser = Grammar(number)
......
......@@ -215,16 +215,16 @@ class StringView(collections.abc.Sized):
return regex.match(self.text, pos=self.begin, endpos=self.end)
def index(self, absolute_index: int) -> int:
"""
Converts an index for a string watched by a StringView object
to an index relative to the string view object, e.g.:
>>> import re
>>> sv = StringView('xxIxx')[2:3]
>>> match = sv.match(re.compile('I'))
>>> match.end()
3
>>> sv.index(match.end())
1
"""Converts an index for a string watched by a StringView object
to an index relative to the string view object, e.g.::
>>> import re
>>> sv = StringView('xxIxx')[2:3]
>>> match = sv.match(re.compile('I'))
>>> match.end()
3
>>> sv.index(match.end())
1
"""
return absolute_index - self.begin
......
This diff is collapsed.
......@@ -25,26 +25,19 @@
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="DHParser 0.8 documentation" href="index.html"/>
<link rel="next" title="Module Reference" href="ModuleReference.html"/>
<link rel="prev" title="DHParser User’s Guide" href="UserGuide.html"/>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="Module Reference" href="ModuleReference.html" />
<link rel="prev" title="DHParser User’s Guide" href="UserGuide.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
......@@ -88,6 +81,7 @@
<p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="StepByStepGuide.html">DHParser’s Step by Step Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="UserGuide.html">DHParser User’s Guide</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">DHParser Reference Manual</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#fundamentals">Fundamentals</a></li>
......@@ -114,7 +108,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">DHParser</a>
......@@ -122,9 +116,10 @@
</nav>
<div class="wy-nav-content">
<div class="rst-content">
......@@ -275,9 +270,7 @@ constitutes an “internal DSL” realized with the means of the Python language
</div>
<div class="articleComments">
</div>
</div>
<footer>
......@@ -299,7 +292,7 @@ constitutes an “internal DSL” realized with the means of the Python language
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
......@@ -318,6 +311,7 @@ constitutes an “internal DSL” realized with the means of the Python language
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'0.8',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
......@@ -336,14 +330,13 @@ constitutes an “internal DSL” realized with the means of the Python language
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
SphinxRtdTheme.Navigation.enableSticky();
});
</script>
</script>
</body>
</html>
\ No newline at end of file
This diff is collapsed.
......@@ -25,26 +25,19 @@
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="DHParser 0.8 documentation" href="index.html"/>
<link rel="next" title="DHParser Reference Manual" href="ReferenceManual.html"/>
<link rel="prev" title="Welcome to DHParser’s documentation!" href="index.html"/>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="DHParser Reference Manual" href="ReferenceManual.html" />
<link rel="prev" title="DHParser’s Step by Step Guide" href="StepByStepGuide.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
......@@ -88,6 +81,7 @@
<p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="StepByStepGuide.html">DHParser’s Step by Step Guide</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">DHParser User’s Guide</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#introduction">Introduction</a></li>
</ul>
......@@ -105,7 +99,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">DHParser</a>
......@@ -113,9 +107,10 @@
</nav>
<div class="wy-nav-content">
<div class="rst-content">
......@@ -181,9 +176,7 @@ assumes an XML-based workflow.</p>
</div>
<div class="articleComments">
</div>
</div>
<footer>
......@@ -192,7 +185,7 @@ assumes an XML-based workflow.</p>
<a href="ReferenceManual.html" class="btn btn-neutral float-right" title="DHParser Reference Manual" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="index.html" class="btn btn-neutral" title="Welcome to DHParser’s documentation!" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
<a href="StepByStepGuide.html" class="btn btn-neutral" title="DHParser’s Step by Step Guide" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
......@@ -205,7 +198,7 @@ assumes an XML-based workflow.</p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
......@@ -224,6 +217,7 @@ assumes an XML-based workflow.</p>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'0.8',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
......@@ -242,14 +236,13 @@ assumes an XML-based workflow.</p>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
SphinxRtdTheme.Navigation.enableSticky();
});
</script>
</script>
</body>
</html>
\ No newline at end of file
......@@ -292,7 +292,8 @@
<div class="viewcode-block" id="Compiler.method_name"><a class="viewcode-back" href="../ModuleReference.html#compile.Compiler.method_name">[docs]</a> <span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">method_name</span><span class="p">(</span><span class="n">node_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="sd">&quot;&quot;&quot;Returns the method name for `node_name`, e.g.::</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the method name for `node_name`, e.g.::</span>
<span class="sd"> &gt;&gt;&gt; Compiler.method_name(&#39;expression&#39;)</span>
<span class="sd"> &#39;on_expression&#39;</span>
......
......@@ -225,52 +225,53 @@
<div class="viewcode-block" id="EBNFGrammar"><a class="viewcode-back" href="../ModuleReference.html#ebnf.EBNFGrammar">[docs]</a><span class="k">class</span> <span class="nc">EBNFGrammar</span><span class="p">(</span><span class="n">Grammar</span><span class="p">):</span>
<span class="sa">r</span><span class="sd">&quot;&quot;&quot;Parser for an EBNF source file, with this grammar:</span>
<span class="sd"> # EBNF-Grammar in EBNF</span>
<span class="sd"> @ comment = /#.*(?:\n|$)/ # comments start with &#39;#&#39; and eat all chars up to and including &#39;\n&#39;</span>
<span class="sd"> @ whitespace = /\s*/ # whitespace includes linefeed</span>
<span class="sd"> @ literalws = right # trailing whitespace of literals will be ignored tacitly</span>
<span class="sd"> syntax = [~//] { definition | directive } §EOF</span>
<span class="sd"> definition = symbol §&quot;=&quot; expression</span>
<span class="sd"> directive = &quot;@&quot; §symbol &quot;=&quot; ( regexp | literal | list_ )</span>
<span class="sd"> expression = term { &quot;|&quot; term }</span>
<span class="sd"> term = { [&quot;§&quot;] factor }+ # &quot;§&quot; means all following factors mandatory</span>
<span class="sd"> factor = [flowmarker] [retrieveop] symbol !&quot;=&quot; # negative lookahead to be sure it&#39;s not a definition</span>
<span class="sd"> | [flowmarker] literal</span>
<span class="sd"> | [flowmarker] plaintext</span>
<span class="sd"> | [flowmarker] regexp</span>
<span class="sd"> | [flowmarker] whitespace</span>
<span class="sd"> | [flowmarker] oneormore</span>
<span class="sd"> | [flowmarker] group</span>
<span class="sd"> | [flowmarker] unordered</span>
<span class="sd"> | repetition</span>
<span class="sd"> | option</span>
<span class="sd"> flowmarker = &quot;!&quot; | &quot;&amp;&quot; # &#39;!&#39; negative lookahead, &#39;&amp;&#39; positive lookahead</span>
<span class="sd"> | &quot;-!&quot; | &quot;-&amp;&quot; # &#39;-&#39; negative lookbehind, &#39;-&amp;&#39; positive lookbehind</span>
<span class="sd"> retrieveop = &quot;::&quot; | &quot;:&quot; # &#39;::&#39; pop, &#39;:&#39; retrieve</span>
<span class="sd"> group = &quot;(&quot; §expression &quot;)&quot;</span>
<span class="sd"> unordered = &quot;&lt;&quot; §expression &quot;&gt;&quot; # elements of expression in arbitrary order</span>
<span class="sd"> oneormore = &quot;{&quot; expression &quot;}+&quot;</span>
<span class="sd"> repetition = &quot;{&quot; §expression &quot;}&quot;</span>
<span class="sd"> option = &quot;[&quot; §expression &quot;]&quot;</span>
<span class="sd"> symbol = /(?!\d)\w+/~ # e.g. expression, factor, parameter_list</span>
<span class="sd"> literal = /&quot;(?:[^&quot;]|\\&quot;)*?&quot;/~ # e.g. &quot;(&quot;, &#39;+&#39;, &#39;while&#39;</span>
<span class="sd"> | /&#39;(?:[^&#39;]|\\&#39;)*?&#39;/~ # whitespace following literals will be ignored tacitly.</span>
<span class="sd"> plaintext = /`(?:[^&quot;]|\\&quot;)*?`/~ # like literal but does not eat whitespace</span>
<span class="sd"> regexp = /~?\/(?:\\\/|[^\/])*?\/~?/~ # e.g. /\w+/, ~/#.*(?:\n|$)/~</span>
<span class="sd"> # &#39;~&#39; is a whitespace-marker, if present leading or trailing</span>
<span class="sd"> # whitespace of a regular expression will be ignored tacitly.</span>
<span class="sd"> whitespace = /~/~ # implicit or default whitespace</span>
<span class="sd"> list_ = /\w+/~ { &quot;,&quot; /\w+/~ } # comma separated list of symbols, e.g. BEGIN_LIST, END_LIST,</span>
<span class="sd"> # BEGIN_QUOTE, END_QUOTE ; see CommonMark/markdown.py for an exmaple</span>
<span class="sd"> EOF = !/./</span>
<span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Parser for an EBNF source file, with this grammar::</span>
<span class="sd"> # EBNF-Grammar in EBNF</span>
<span class="sd"> @ comment = /#.*(?:\n|$)/ # comments start with &#39;#&#39; and eat all chars up to and including &#39;\n&#39;</span>
<span class="sd"> @ whitespace = /\s*/ # whitespace includes linefeed</span>
<span class="sd"> @ literalws = right # trailing whitespace of literals will be ignored tacitly</span>
<span class="sd"> syntax = [~//] { definition | directive } §EOF</span>
<span class="sd"> definition = symbol §&quot;=&quot; expression</span>
<span class="sd"> directive = &quot;@&quot; §symbol &quot;=&quot; ( regexp | literal | list_ )</span>
<span class="sd"> expression = term { &quot;|&quot; term }</span>
<span class="sd"> term = { [&quot;§&quot;] factor }+ # &quot;§&quot; means all following factors mandatory</span>
<span class="sd"> factor = [flowmarker] [retrieveop] symbol !&quot;=&quot; # negative lookahead to be sure it&#39;s not a definition</span>
<span class="sd"> | [flowmarker] literal</span>
<span class="sd"> | [flowmarker] plaintext</span>
<span class="sd"> | [flowmarker] regexp</span>
<span class="sd"> | [flowmarker] whitespace</span>
<span class="sd"> | [flowmarker] oneormore</span>
<span class="sd"> | [flowmarker] group</span>
<span class="sd"> | [flowmarker] unordered</span>
<span class="sd"> | repetition</span>
<span class="sd"> | option</span>
<span class="sd"> flowmarker = &quot;!&quot; | &quot;&amp;&quot; # &#39;!&#39; negative lookahead, &#39;&amp;&#39; positive lookahead</span>
<span class="sd"> | &quot;-!&quot; | &quot;-&amp;&quot; # &#39;-&#39; negative lookbehind, &#39;-&amp;&#39; positive lookbehind</span>
<span class="sd"> retrieveop = &quot;::&quot; | &quot;:&quot; # &#39;::&#39; pop, &#39;:&#39; retrieve</span>
<span class="sd"> group = &quot;(&quot; §expression &quot;)&quot;</span>
<span class="sd"> unordered = &quot;&lt;&quot; §expression &quot;&gt;&quot; # elements of expression in arbitrary order</span>
<span class="sd"> oneormore = &quot;{&quot; expression &quot;}+&quot;</span>
<span class="sd"> repetition = &quot;{&quot; §expression &quot;}&quot;</span>
<span class="sd"> option = &quot;[&quot; §expression &quot;]&quot;</span>
<span class="sd"> symbol = /(?!\d)\w+/~ # e.g. expression, factor, parameter_list</span>
<span class="sd"> literal = /&quot;(?:[^&quot;]|\\&quot;)*?&quot;/~ # e.g. &quot;(&quot;, &#39;+&#39;, &#39;while&#39;</span>
<span class="sd"> | /&#39;(?:[^&#39;]|\\&#39;)*?&#39;/~ # whitespace following literals will be ignored tacitly.</span>
<span class="sd"> plaintext = /`(?:[^&quot;]|\\&quot;)*?`/~ # like literal but does not eat whitespace</span>
<span class="sd"> regexp = /~?\/(?:\\\/|[^\/])*?\/~?/~ # e.g. /\w+/, ~/#.*(?:\n|$)/~</span>
<span class="sd"> # &#39;~&#39; is a whitespace-marker, if present leading or trailing</span>
<span class="sd"> # whitespace of a regular expression will be ignored tacitly.</span>
<span class="sd"> whitespace = /~/~ # implicit or default whitespace</span>
<span class="sd"> list_ = /\w+/~ { &quot;,&quot; /\w+/~ } # comma separated list of symbols, e.g. BEGIN_LIST, END_LIST,</span>
<span class="sd"> # BEGIN_QUOTE, END_QUOTE ; see CommonMark/markdown.py for an exmaple</span>
<span class="sd"> EOF = !/./</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">expression</span> <span class="o">=</span> <span class="n">Forward</span><span class="p">()</span>
<span class="n">source_hash__</span> <span class="o">=</span> <span class="s2">&quot;3fc9f5a340f560e847d9af0b61a68743&quot;</span>
......
......@@ -144,7 +144,8 @@
<div itemprop="articleBody">
<h1>All modules for which code is available</h1>
<ul><li><a href="DHParser/syntaxtree.html">DHParser.syntaxtree</a></li>
<ul><li><a href="DHParser/stringview.html">DHParser.stringview</a></li>
<li><a href="DHParser/syntaxtree.html">DHParser.syntaxtree</a></li>
<li><a href="compile.html">compile</a></li>
<li><a href="dsl.html">dsl</a></li>
<li><a href="ebnf.html">ebnf</a></li>
......@@ -152,7 +153,6 @@
<li><a href="log.html">log</a></li>
<li><a href="parse.html">parse</a></li>
<li><a href="preprocess.html">preprocess</a></li>
<li><a href="stringview.html">stringview</a></li>
<li><a href="syntaxtree.html">syntaxtree</a></li>
<li><a href="testing.html">testing</a></li>
<li><a href="toolkit.html">toolkit</a></li>
......
......@@ -572,8 +572,8 @@
<span class="n">log_file_name</span> <span class="o">=</span> <span class="n">log_file_name</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span>
<span class="n">full_history</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&lt;h1&gt;Full parsing history of &quot;</span><span class="si">%s</span><span class="s1">&quot;&lt;/h1&gt;&#39;</span> <span class="o">%</span> <span class="n">log_file_name</span><span class="p">]</span> <span class="c1"># type: List[str]</span>
<span class="n">match_history</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&lt;h1&gt;Match history of parsing &quot;</span><span class="si">%s</span><span class="s1">&quot;&lt;/h1&gt;&#39;</span> <span class="o">%</span> <span class="n">log_file_name</span><span class="p">]</span> <span class="c1"># type: List[str]</span>
<span class="n">errors_only</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;&lt;h1&gt;Errors when parsing &quot;</span><span class="si">%s</span><span class="s1">&quot;&lt;/h1&gt;&#39;</span> <span class="o">%</span> <span class="n">log_file_name</span><span class="p">]</span> <span class="c1"># type: List[str]</span>
<span class="c1"># match_history = [&#39;&lt;h1&gt;Match history of parsing &quot;%s&quot;&lt;/h1&gt;&#39; % log_file_name] # type: List[str]</span>
<span class="c1"># errors_only = [&#39;&lt;h1&gt;Errors when parsing &quot;%s&quot;&lt;/h1&gt;&#39; % log_file_name] # type: List[str]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">grammar</span><span class="o">.</span><span class="n">history__</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">LOG_SIZE_THRESHOLD</span><span class="p">:</span>
<span class="n">warning</span> <span class="o">=</span><span class="p">(</span><span class="s1">&#39;Sorry, man, </span><span class="si">%i</span><span class="s1">K history records is just too many! &#39;</span>
......@@ -581,29 +581,29 @@
<span class="o">%</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">grammar</span><span class="o">.</span><span class="n">history__</span><span class="p">)</span><span class="o">//</span><span class="mi">1000</span><span class="p">,</span> <span class="n">LOG_SIZE_THRESHOLD</span><span class="o">//</span><span class="mi">1000</span><span class="p">))</span>
<span class="n">html_warning</span> <span class="o">=</span> <span class="s1">&#39;&lt;p&gt;&lt;strong&gt;&#39;</span> <span class="o">+</span> <span class="n">warning</span> <span class="o">+</span> <span class="s1">&#39;&lt;/strong&gt;&lt;/p&gt;&#39;</span>
<span class="n">full_history</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">html_warning</span><span class="p">)</span>
<span class="n">match_history</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">html_warning</span><span class="p">)</span>
<span class="n">errors_only</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">html_warning</span><span class="p">)</span>
<span class="c1"># match_history.append(html_warning)</span>
<span class="c1"># errors_only.append(html_warning)</span>
<span class="n">lead_in</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span> <span class="n">join</span><span class="p">([</span><span class="s1">&#39;&lt;table&gt;&#39;</span><span class="p">,</span> <span class="n">HistoryRecord</span><span class="o">.</span><span class="n">COLGROUP</span><span class="p">,</span> <span class="n">HistoryRecord</span><span class="o">.</span><span class="n">HEADINGS</span><span class="p">])</span>
<span class="n">full_history</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lead_in</span><span class="p">)</span>
<span class="n">match_history</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lead_in</span><span class="p">)</span>
<span class="n">errors_only</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lead_in</span><span class="p">)</span>
<span class="c1"># match_history.append(lead_in)</span>
<span class="c1"># errors_only.append(lead_in)</span>
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">grammar</span><span class="o">.</span><span class="n">history__</span><span class="p">[</span><span class="o">-</span><span class="n">LOG_SIZE_THRESHOLD</span><span class="p">:]:</span>
<span class="n">line</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">as_html_tr</span><span class="p">()</span> <span class="k">if</span> <span class="n">html</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">record</span><span class="p">)</span>
<span class="n">append_line</span><span class="p">(</span><span class="n">full_history</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
<span class="k">if</span> <span class="n">record</span><span class="o">.</span><span class="n">node</span> <span class="ow">and</span> <span class="n">record</span><span class="o">.</span><span class="n">node</span><span class="o">.</span><span class="n">parser</span><span class="o">.</span><span class="n">ptype</span> <span class="o">!=</span> <span class="n">WHITESPACE_PTYPE</span><span class="p">:</span>
<span class="n">append_line</span><span class="p">(</span><span class="n">match_history</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
<span class="k">if</span> <span class="n">record</span><span class="o">.</span><span class="n">node</span><span class="o">.</span><span class="n">errors</span><span class="p">:</span>
<span class="n">append_line</span><span class="p">(</span><span class="n">errors_only</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
<span class="c1"># if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:</span>
<span class="c1"># append_line(match_history, line)</span>
<span class="c1"># if record.node.errors:</span>
<span class="c1"># append_line(errors_only, line)</span>
<span class="n">write_log</span><span class="p">(</span><span class="n">full_history</span><span class="p">,</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="s1">&#39;_full&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">full_history</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">LOG_TAIL_THRESHOLD</span> <span class="o">+</span> <span class="mi">10</span><span class="p">:</span>
<span class="n">heading</span> <span class="o">=</span> <span class="s1">&#39;&lt;h1&gt;Last 500 records of parsing history of &quot;</span><span class="si">%s</span><span class="s1">&quot;&lt;/h1&gt;&#39;</span> <span class="o">%</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="n">lead_in</span>
<span class="n">write_log</span><span class="p">([</span><span class="n">heading</span><span class="p">]</span> <span class="o">+</span> <span class="n">full_history</span><span class="p">[</span><span class="o">-</span><span class="n">LOG_TAIL_THRESHOLD</span><span class="p">:],</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="s1">&#39;_full.tail&#39;</span><span class="p">)</span>
<span class="n">write_log</span><span class="p">(</span><span class="n">match_history</span><span class="p">,</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="s1">&#39;_match&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">errors_only</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">3</span> <span class="ow">or</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">grammar</span><span class="o">.</span><span class="n">history__</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">LOG_SIZE_THRESHOLD</span>
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">errors_only</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">2</span><span class="p">)):</span>
<span class="n">write_log</span><span class="p">(</span><span class="n">errors_only</span><span class="p">,</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="s1">&#39;_errors&#39;</span><span class="p">)</span></div>
<span class="n">write_log</span><span class="p">([</span><span class="n">heading</span><span class="p">]</span> <span class="o">+</span> <span class="n">full_history</span><span class="p">[</span><span class="o">-</span><span class="n">LOG_TAIL_THRESHOLD</span><span class="p">:],</span> <span class="n">log_file_name</span> <span class="o">+</span> <span class="s1">&#39;_full.tail&#39;</span><span class="p">)</span></div>
<span class="c1"># write_log(match_history, log_file_name + &#39;_match&#39;)</span>
<span class="c1"># if (len(errors_only) &gt; 3 or (len(grammar.history__) &lt;= LOG_SIZE_THRESHOLD</span>
<span class="c1"># and len(errors_only) &gt; 2)):</span>
<span class="c1"># write_log(errors_only, log_file_name + &#39;_errors&#39;)</span>
</pre></div>
</div>
......
......@@ -186,7 +186,7 @@
<span class="kn">from</span> <span class="nn">DHParser.preprocess</span> <span class="k">import</span> <span class="n">BEGIN_TOKEN</span><span class="p">,</span> <span class="n">END_TOKEN</span><span class="p">,</span> <span class="n">RX_TOKEN_NAME</span>
<span class="kn">from</span> <span class="nn">DHParser.stringview</span> <span class="k">import</span> <span class="n">StringView</span><span class="p">,</span> <span class="n">EMPTY_STRING_VIEW</span>
<span class="kn">from</span> <span class="nn">DHParser.syntaxtree</span> <span class="k">import</span> <span class="n">Node</span><span class="p">,</span> <span class="n">RootNode</span><span class="p">,</span> <span class="n">ParserBase</span><span class="p">,</span> <span class="n">WHITESPACE_PTYPE</span><span class="p">,</span> \
<span class="n">TOKEN_PTYPE</span><span class="p">,</span> <span class="n">ZOMBIE_PARSER</span>
<span class="n">PLAINTEXT_PTYPE</span><span class="p">,</span> <span class="n">TOKEN_PTYPE</span><span class="p">,</span> <span class="n">ZOMBIE_PARSER</span>
<span class="kn">from</span> <span class="nn">DHParser.toolkit</span> <span class="k">import</span> <span class="n">sane_parser_name</span><span class="p">,</span> <span class="n">escape_control_characters</span><span class="p">,</span> <span class="n">re</span><span class="p">,</span> <span class="n">typing</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="k">import</span> <span class="n">Callable</span><span class="p">,</span> <span class="n">cast</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">DefaultDict</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Set</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">Optional</span>
......@@ -502,7 +502,7 @@
<span class="sd"> (&quot;direct instantiation&quot;), or by assigning the root parser to the</span>
<span class="sd"> class variable &quot;root__&quot; of a descendant class of class Grammar.</span>
<span class="sd"> Example for direct instantiation of a grammar:</span>
<span class="sd"> Example for direct instantiation of a grammar::</span>
<span class="sd"> &gt;&gt;&gt; number = RE(&#39;\d+&#39;) + RE(&#39;\.&#39;) + RE(&#39;\d+&#39;) | RE(&#39;\d+&#39;)</span>
<span class="sd"> &gt;&gt;&gt; number_parser = Grammar(number)</span>
......@@ -1042,6 +1042,7 @@
<span class="sd"> &gt;&gt;&gt; Grammar(while_token)(&quot;while&quot;).content</span>
<span class="sd"> &#39;while&#39;</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">assert</span> <span class="n">PLAINTEXT_PTYPE</span> <span class="o">==</span> <span class="s2">&quot;:PlainText&quot;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
......
This diff is collapsed.
This diff is collapsed.
......@@ -455,13 +455,13 @@
<span class="k">except</span> <span class="n">UnknownParserError</span> <span class="k">as</span> <span class="n">upe</span><span class="p">:</span>
<span class="n">cst</span> <span class="o">=</span> <span class="n">cst</span><span class="o">.</span><span class="n">new_error</span><span class="p">(</span><span class="n">Node</span><span class="p">(</span><span class="n">ZOMBIE_PARSER</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">init_pos</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">upe</span><span class="p">))</span>
<span class="n">clean_test_name</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">test_name</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;*&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">log_ST</span><span class="p">(</span><span class="n">cst</span><span class="p">,</span> <span class="s2">&quot;match_</span><span class="si">%s</span><span class="s2">_</span><span class="si">%s</span><span class="s2">.cst&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">parser_name</span><span class="p">,</span> <span class="n">clean_test_name</span><span class="p">))</span>
<span class="c1"># log_ST(cst, &quot;match_%s_%s.cst&quot; % (parser_name, clean_test_name))</span>
<span class="n">tests</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;__cst__&#39;</span><span class="p">,</span> <span class="p">{})[</span><span class="n">test_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">cst</span>
<span class="k">if</span> <span class="s2">&quot;ast&quot;</span> <span class="ow">in</span> <span class="n">tests</span> <span class="ow">or</span> <span class="n">report</span><span class="p">:</span>
<span class="n">ast</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">cst</span><span class="p">)</span>
<span class="n">transform</span><span class="p">(</span><span class="n">ast</span><span class="p">)</span>
<span class="n">tests</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;__ast__&#39;</span><span class="p">,</span> <span class="p">{})[</span><span class="n">test_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">ast</span>
<span class="n">log_ST</span><span class="p">(</span><span class="n">ast</span><span class="p">,</span> <span class="s2">&quot;match_</span><span class="si">%s</span><span class="s2">_</span><span class="si">%s</span><span class="s2">.ast&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">parser_name</span><span class="p">,</span> <span class="n">clean_test_name</span><span class="p">))</span>
<span class="c1"># log_ST(ast, &quot;match_%s_%s.ast&quot; % (parser_name, clean_test_name))</span>
<span class="k">if</span> <span class="n">is_error</span><span class="p">(</span><span class="n">cst</span><span class="o">.</span><span class="n">error_flag</span><span class="p">):</span>
<span class="n">errors</span> <span class="o">=</span> <span class="n">adjust_error_locations</span><span class="p">(</span><span class="n">cst</span><span class="o">.</span><span class="n">collect_errors</span><span class="p">(),</span> <span class="n">test_code</span><span class="p">)</span>
<span class="n">errata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;Match test &quot;</span><span class="si">%s</span><span class="s1">&quot; for parser &quot;</span><span class="si">%s</span><span class="s1">&quot; failed:</span><span class="se">\n\t</span><span class="s1">Expr.: </span><span class="si">%s</span><span class="se">\n\n\t</span><span class="si">%s</span><span class="se">\n\n</span><span class="s1">&#39;</span> <span class="o">%</span>
......
This diff is collapsed.
......@@ -19,14 +19,14 @@ However, in order to add or change the source code of DHParser, its module
structure must be understood. DHParser's modules can roughly be sorted into
three different categories:
1. Modules that contain the basic funcionality for packrat-parsing,
1. Modules that contain the basic functionality for packrat-parsing,
AST-transformation and the skeleton for a DSL-compilers.
2. Modules for EBNF-Grammars and DSL compilation.
3. Support or "toolkit"-modules that contain different helpful functions
The import-order of DHParser's modules runs accross these categories. In the
The import-order of DHParser's modules runs across these categories. In the
following list the modules further below in the list may import one or
more of the modules further above in the list, but not the other way round:
......@@ -44,14 +44,14 @@ more of the modules further above in the list, but not the other way round:
- syntaxtree.py -- syntax tree classes for DHParser
- transform.py -- transformation functions for converting the concrete
into the abstract syntax tree
into the abstract syntax tree
- logging.py -- logging and debugging for DHParser
- parse.py -- parser combinators for for DHParser
- compile.py -- abstract base class for compilers that transform an AST
into something useful
into something useful
- ebnf.py -- EBNF -> Python-Parser compilation for DHParser
......
......@@ -3,6 +3,8 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
.. default-domain::py