- extraced classes Error, StringView, ParserBase, MockParser and ZombieParser to new module base.py

DHParser/__init__.py

@@ -18,6 +18,7 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
+from .base import *
 from .dsl import *
 from .ebnf import *
 from .parser import *
@@ -30,4 +31,4 @@ from .versionnumber import __version__
 
 __author__ = "Eckhart Arnold <arnold@badw.de>"
 __copyright__ = "http://www.apache.org/licenses/LICENSE-2.0"
-# __all__ = ['toolkit', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber']  # flat namespace
+# __all__ = ['toolkit', 'base', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber']  # flat namespace

DHParser/dsl.py

@@ -15,7 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 implied.  See the License for the specific language governing
 permissions and limitations under the License.
 
-Module ``DSLsupport`` contains various functions to support the
+Module ``dsl`` contains various functions to support the
 compilation of domain specific languages based on an EBNF-grammar.
 """
 
@@ -35,7 +35,8 @@ from DHParser.ebnf import EBNFCompiler, grammar_changed, \
     PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
 from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object
 from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
-from DHParser.syntaxtree import Error, is_error, has_errors, only_errors, Node, TransformationFunc
+from DHParser.syntaxtree import Node, TransformationFunc
+from DHParser.base import Error, is_error, has_errors, only_errors
 
 __all__ = ('GrammarError',
            'CompilationError',

DHParser/ebnf.py

@@ -33,7 +33,8 @@ from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name
 from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, RE, NegativeLookahead, \
     Alternative, Series, Option, Required, OneOrMore, ZeroOrMore, Token, Compiler, \
     PreprocessorFunc
-from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Error, Node, TransformationFunc
+from DHParser.syntaxtree import Node, TransformationFunc
+from DHParser.base import WHITESPACE_PTYPE, TOKEN_PTYPE, Error
 from DHParser.transform import TransformationDict, traverse, remove_brackets, \
     reduce_single_child, replace_by_single_child, remove_expendables, \
     remove_tokens, flatten, forbid, assert_content, remove_infix_operator

DHParser/parser.py

@@ -75,10 +75,10 @@ except ImportError:
     from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union, Optional
 
 from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name
-from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, ParserBase, \
-    Error, is_error, has_errors, Node, TransformationFunc
-from DHParser.toolkit import StringView, EMPTY_STRING_VIEW, sv_match, sv_index, sv_search, \
-    load_if_file, error_messages, line_col
+from DHParser.syntaxtree import Node, TransformationFunc
+from DHParser.base import ParserBase, WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, Error, is_error, has_errors, \
+    StringView, EMPTY_STRING_VIEW
+from DHParser.toolkit import load_if_file, error_messages, line_col
 
 __all__ = ('PreprocessorFunc',
            'HistoryRecord',
@@ -1066,9 +1066,9 @@ class RegExp(Parser):
         return RegExp(regexp, self.name)
 
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
-        match = text[0:1] != BEGIN_TOKEN and sv_match(self.regexp, text)  # ESC starts a preprocessor token.
+        match = text[0:1] != BEGIN_TOKEN and text.match(self.regexp)  # ESC starts a preprocessor token.
         if match:
-            end = sv_index(match.end(), text)
+            end = text.index(match.end())
             return Node(self, text[:end]), text[end:]
         return None, text
 
@@ -1521,8 +1521,8 @@ class Required(FlowOperator):
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
         node, text_ = self.parser(text)
         if not node:
-            m = sv_search(Required.RX_ARGUMENT, text)  # re.search(r'\s(\S)', text)
-            i = max(1, sv_index(m.regs[1][0], text)) if m else 1
+            m = text.search(Required.RX_ARGUMENT)  # re.search(r'\s(\S)', text)
+            i = max(1, text.index(m.regs[1][0])) if m else 1
             node = Node(self, text[:i])
             text_ = text[i:]
             # assert False, "*"+text[:i]+"*"
@@ -1585,7 +1585,7 @@ class Lookbehind(FlowOperator):
 
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
         backwards_text = self.grammar.reversed__[len(text):]  # self.grammar.document__[-len(text) - 1::-1]
-        if self.sign(sv_match(self.regexp, backwards_text)):
+        if self.sign(backwards_text.match(self.regexp)):
             return Node(self, ''), text
         else:
             return None, text

DHParser/syntaxtree.py

@@ -32,155 +32,14 @@ except ImportError:
     from .typing34 import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \
         Iterator, Iterable, List, NamedTuple, Sequence, Union, Text, Tuple, Hashable
 
-from DHParser.toolkit import is_logging, log_dir, StringView, linebreaks, line_col, identity
-
-__all__ = ('WHITESPACE_PTYPE',
-           'MockParser',
-           'TOKEN_PTYPE',
-           'ZOMBIE_PARSER',
-           'ParserBase',
-           'Error',
-           'is_warning',
-           'is_error',
-           'has_errors',
-           'Node',
+from DHParser.toolkit import is_logging, log_dir, linebreaks, line_col, identity
+from DHParser.base import MockParser, ZOMBIE_PARSER, Error, StringView
+
+__all__ = ('Node',
            'mock_syntax_tree',
            'TransformationFunc')
 
 
-class ParserBase:
-    """
-    ParserBase is the base class for all real and mock parser classes.
-    It is defined here, because Node objects require a parser object
-    for instantiation.
-    """
-    def __init__(self, name=''):  # , pbases=frozenset()):
-        self.name = name  # type: str
-        self._ptype = ':' + self.__class__.__name__  # type: str
-
-    def __repr__(self):
-        return self.name + self.ptype
-
-    def __str__(self):
-        return self.name + (' = ' if self.name else '') + repr(self)
-
-    @property
-    def ptype(self) -> str:
-        return self._ptype
-
-    @property
-    def repr(self) -> str:
-        return self.name if self.name else repr(self)
-
-
-WHITESPACE_PTYPE = ':Whitespace'
-TOKEN_PTYPE = ':Token'
-
-
-class MockParser(ParserBase):
-    """
-    MockParser objects can be used to reconstruct syntax trees from a
-    serialized form like S-expressions or XML. Mock objects can mimic
-    different parser types by assigning them a ptype on initialization.
-    
-    Mock objects should not be used for anything other than 
-    syntax tree (re-)construction. In all other cases where a parser
-    object substitute is needed, chose the singleton ZOMBIE_PARSER.
-    """
-    def __init__(self, name='', ptype=''):  # , pbases=frozenset()):
-        assert not ptype or ptype[0] == ':'
-        super(MockParser, self).__init__(name)
-        self.name = name
-        self._ptype = ptype or ':' + self.__class__.__name__
-
-
-class ZombieParser(MockParser):
-    """
-    Serves as a substitute for a Parser instance.
-
-    ``ZombieParser`` is the class of the singelton object
-    ``ZOMBIE_PARSER``. The  ``ZOMBIE_PARSER`` has a name and can be
-    called, but it never matches. It serves as a substitute where only
-    these (or one of these properties) is needed, but no real Parser-
-    object is instantiated.
-    """
-    alive = False
-
-    def __init__(self):
-        super(ZombieParser, self).__init__("__ZOMBIE__")
-        assert not self.__class__.alive, "There can be only one!"
-        assert self.__class__ == ZombieParser, "No derivatives, please!"
-        self.__class__.alive = True
-
-    def __copy__(self):
-        return self
-
-    def __deepcopy__(self, memo):
-        return self
-
-    def __call__(self, text):
-        """Better call Saul ;-)"""
-        return None, text
-
-
-ZOMBIE_PARSER = ZombieParser()
-
-
-class Error:
-    __slots__ = ['message', 'level', 'code', 'pos', 'line', 'column']
-
-    WARNING   = 1
-    ERROR     = 1000
-    HIGHEST   = ERROR
-
-    def __init__(self, message: str, level: int=ERROR, code: Hashable=0):
-        self.message = message
-        assert level >= 0
-        self.level = level or Error.ERROR
-        self.code = code
-        self.pos = -1
-        self.line = -1
-        self.column = -1
-
-    def __str__(self):
-        prefix = ''
-        if self.line > 0:
-            prefix = "line: %3i, column: %2i, " % (self.line, self.column)
-        return prefix + "%s: %s" % (self.level_str, self.message)
-
-    @property
-    def level_str(self):
-        return "Warning" if is_warning(self.level) else "Error"
-
-
-def is_warning(level: int) -> bool:
-    return level < Error.ERROR
-
-
-def is_error(level:  int) -> bool:
-    return level >= Error.ERROR
-
-
-def has_errors(messages: Iterable[Error], level: int=Error.ERROR) -> bool:
-    """
-    Returns True, if at least one entry in `messages` has at
-    least the given error `level`.
-    """
-    for err_obj in messages:
-        if err_obj.level >= level:
-            return True
-    return False
-
-
-def only_errors(messages: Iterable[Error], level: int=Error.ERROR) -> Iterator[Error]:
-    """
-    Returns an Iterator that yields only those messages that have
-    at least the given error level.
-    """
-    return (err for err in messages if err.level >= level)
-
-
-
 ChildrenType = Tuple['Node', ...]
 StrictResultType = Union[ChildrenType, StringView, str]
 ResultType = Union[ChildrenType, 'Node', StringView, str, None]
@@ -347,7 +206,7 @@ class Node(collections.abc.Sized):
         return self._errors.copy()
 
 
-    def add_error(self, message: str, level: int=Error.ERROR, code: Hashable=0) -> 'Node':
+    def add_error(self, message: str, level: int= Error.ERROR, code: Hashable=0) -> 'Node':
         self._errors.append(Error(message, level, code))
         self.error_flag = max(self.error_flag, self._errors[-1].level)
         return self

DHParser/testing.py

@@ -28,7 +28,8 @@ except ImportError:
     import re
 
 from DHParser.toolkit import is_logging, clear_logs, error_messages
-from DHParser.syntaxtree import is_error, mock_syntax_tree, flatten_sxpr
+from DHParser.syntaxtree import mock_syntax_tree, flatten_sxpr
+from DHParser.base import is_error
 
 __all__ = ('unit_from_configfile',
            'unit_from_json',

DHParser/toolkit.py

@@ -32,12 +32,13 @@ already exists.
 
 import bisect
 import codecs
-import collections
 import contextlib
 import functools
 import hashlib
 import os
 
+from DHParser.base import StringView
+
 try:
     import regex as re
 except ImportError:
@@ -53,10 +54,6 @@ __all__ = ('logging',
            'is_logging',
            'log_dir',
            'logfile_basename',
-           'StringView',
-           'sv_match',
-           'sv_index',
-           'sv_search',
            'linebreaks',
            'line_col',
            'error_messages',
@@ -154,124 +151,6 @@ def clear_logs(logfile_types={'.cst', '.ast', '.log'}):
         os.rmdir(log_dirname)
 
 
-class StringView(collections.abc.Sized):
-    """"A rudimentary StringView class, just enough for the use cases
-    in parser.py.
-
-    Slicing Python-strings always yields copies of a segment of the original
-    string. See: https://mail.python.org/pipermail/python-dev/2008-May/079699.html
-    However, this becomes costly (in terms of space and as a consequence also
-    time) when parsing longer documents. Unfortunately, Python's `memoryview`
-    does not work for unicode strings. Hence, the StringView class.
-    """
-
-    __slots__ = ['text', 'begin', 'end', 'len', 'fullstring_flag']
-
-    def __init__(self, text: str, begin: Optional[int] = 0, end: Optional[int] = None) -> None:
-        self.text = text  # type: str
-        self.begin = 0  # type: int
-        self.end = 0    # type: int
-        self.begin, self.end = StringView.real_indices(begin, end, len(text))
-        self.len = max(self.end - self.begin, 0)
-        self.fullstring_flag = (self.begin == 0 and self.len == len(self.text))
-
-    @staticmethod
-    def real_indices(begin, end, len):
-        def pack(index, len):
-            index = index if index >= 0 else index + len
-            return 0 if index < 0 else len if index > len else index
-        if begin is None:  begin = 0
-        if end is None:  end = len
-        return pack(begin, len), pack(end, len)
-
-    def __bool__(self):
-        return bool(self.text) and self.end > self.begin
-
-    def __len__(self):
-        return self.len
-
-    def __str__(self):
-        if self.fullstring_flag:  # optimization: avoid slicing/copying
-            return self.text
-        return self.text[self.begin:self.end]
-
-    def __getitem__(self, index):
-        # assert isinstance(index, slice), "As of now, StringView only allows slicing."
-        # assert index.step is None or index.step == 1, \
-        #     "Step sizes other than 1 are not yet supported by StringView"
-        start, stop = StringView.real_indices(index.start, index.stop, self.len)
-        return StringView(self.text, self.begin + start, self.begin + stop)
-
-    def __eq__(self, other):
-        return str(self) == str(other)  # PERFORMANCE WARNING: This creates copies of the strings
-
-    def count(self, sub, start=None, end=None) -> int:
-        if self.fullstring_flag:
-            return self.text.count(sub, start, end)
-        elif start is None and end is None:
-            return self.text.count(sub, self.begin, self.end)
-        else:
-            start, end = StringView.real_indices(start, end, self.len)
-            return self.text.count(sub, self.begin + start, self.begin + end)
-
-    def find(self, sub, start=None, end=None) -> int:
-        if self.fullstring_flag:
-            return self.text.find(sub, start, end)
-        elif start is None and end is None:
-            return self.text.find(sub, self.begin, self.end) - self.begin
-        else:
-            start, end = StringView.real_indices(start, end, self.len)
-            return self.text.find(sub, self.begin + start, self.begin + end) - self.begin
-
-    def rfind(self, sub, start=None, end=None) -> int:
-        if self.fullstring_flag:
-            return self.text.rfind(sub, start, end)
-        if start is None and end is None:
-            return self.text.rfind(sub, self.begin, self.end) - self.begin
-        else:
-            start, end = StringView.real_indices(start, end, self.len)
-            return self.text.rfind(sub, self.begin + start, self.begin + end) - self.begin
-
-    def startswith(self, prefix: str, start:int = 0, end:Optional[int] = None) -> bool:
-        start += self.begin
-        end = self.end if end is None else self.begin + end
-        return self.text.startswith(prefix, start, end)
-
-
-
-def sv_match(regex, sv: StringView):
-    return regex.match(sv.text, pos=sv.begin, endpos=sv.end)
-
-
-def sv_index(absolute_index: int, sv: StringView) -> int:
-    """
-    Converts the an index into string watched by a StringView object
-    to an index relativ to the string view object, e.g.:
-    >>> sv = StringView('xxIxx')[2:3]
-    >>> match = sv_match(re.compile('I'), sv)
-    >>> match.end()
-    3
-    >>> sv_index(match.end(), sv)
-    1
-    """
-    return absolute_index - sv.begin
-
-
-def sv_indices(absolute_indices: Iterable[int], sv: StringView) -> Tuple[int, ...]:
-    """Converts indices into a string watched by a StringView object
-    to an index relativ to the string view object. See also: `sv_index()`
-    """
-    return tuple(index - sv.begin for index in absolute_indices)
-
-
-def sv_search(regex, sv: StringView):
-    return regex.search(sv.text, pos=sv.begin, endpos=sv.end)
-
-
-
-EMPTY_STRING_VIEW = StringView('')
-
-
 def linebreaks(text: Union[StringView, str]):
     lb = [-1]
     i = text.find('\n', 0)
@@ -295,14 +174,14 @@ def line_col(text: Union[StringView, str], pos: int) -> Tuple[int, int]:
 
 
 @line_col.register(list)
-def _line_col(linebreaks: List[int], pos: int) -> Tuple[int, int]:
+def _line_col(lbreaks: List[int], pos: int) -> Tuple[int, int]:
     """Returns the position within a text as (line, column)-tuple based
     on a list of all line breaks, including -1 and EOF.
     """
-    if pos < 0 or pos > linebreaks[-1]:  # one character behind EOF is still an allowed position!
-        raise ValueError('Position %i outside text of length %s !' % (pos, linebreaks[-1]))
-    line = bisect.bisect_left(linebreaks, pos)
-    column = pos - linebreaks[line-1]
+    if pos < 0 or pos > lbreaks[-1]:  # one character behind EOF is still an allowed position!
+        raise ValueError('Position %i outside text of length %s !' % (pos, lbreaks[-1]))
+    line = bisect.bisect_left(lbreaks, pos)
+    column = pos - lbreaks[line - 1]
     return line, column
 
 

DHParser/transform.py

@@ -20,7 +20,8 @@ permissions and limitations under the License.
 import inspect
 from functools import partial, reduce, singledispatch
 
-from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, MockParser, Node
+from DHParser.syntaxtree import Node
+from DHParser.base import WHITESPACE_PTYPE, TOKEN_PTYPE, MockParser
 
 try:
     import regex as re

test/test_base.py

+#!/usr/bin/python3
+
+"""test_base.py - tests of the base-module of DHParser
+
+
+Author: Eckhart Arnold <arnold@badw.de>
+
+Copyright 2017 Bavarian Academy of Sciences and Humanities
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+try:
+    import regex as re
+except ImportError:
+    import re
+
+from DHParser import StringView, EMPTY_STRING_VIEW
+
+
+class TestStringView:
+    def test_real_indices(self):
+        assert StringView.real_indices(3, 5, 10) == (3, 5)
+        assert StringView.real_indices(None, None, 10) == (0, 10)
+        assert StringView.real_indices(-2, -1, 10) == (8, 9)
+        assert StringView.real_indices(-3, 11, 10) == (7, 10)
+        assert StringView.real_indices(-5, -12, 10) == (5, 0)
+        assert StringView.real_indices(-12, -5, 10) == (0, 5)
+        assert StringView.real_indices(7, 6, 10) == (7, 6)
+        assert StringView.real_indices(None, 0, 10) == (0, 0)
+
+    def test_creation(self):
+        s = "0123456789"
+        assert str(StringView(s)) == s
+        assert str(StringView(s, 3, 4)) == '3'
+        assert str(StringView(s, -4)) == '6789'
+
+    def test_equality(self):
+        s = "0123456789"
+        assert StringView(s) == s
+        assert StringView(s, 3, 4) == '3'
+        assert StringView(s, -4) == '6789'
+
+    def test_slicing(self):
+        s = " 0123456789 "
+        sv = StringView(s, 1, -1)
+        assert sv == '0123456789'
+        assert sv[3:4] == '3'
+        assert sv[-3:-1] == '78'
+        assert sv[4:3] == ''
+        assert sv[:4] == '0123'
+        assert sv[4:] == '456789'
+        assert sv[-2:] == '89'
+        assert sv[:-5] == '01234'
+        assert isinstance(sv[3:5], StringView)
+
+    def test_len(self):
+        s = " 0123456789 "
+        sv = StringView(s, 1, -1)
+        assert len(sv) == 10
+        assert sv.len == 10
+        assert len(sv[5:5]) == 0
+        assert len(sv[7:4]) == 0
+        assert len(sv[-12:-2]) == 8
+        assert len(sv[-12:12]) == 10
+
+    def test_bool(self):
+        assert not StringView('')
+        assert StringView('x')
+        s = " 0123456789 "
+        sv = StringView(s, 1, -1)
+        assert not sv[5:4]
+        assert sv[4:5], str(sv[4:5])
+        assert not sv[3:3]
+        assert not sv[12:13]
+        assert sv[0:20]
+
+    def test_sv_match(self):
+        s = " 0123456789 "
+        sv = StringView(s, 1, -1)
+        assert sv.match(re.compile(r'\d'))