Commit 802fcc4a authored by Eckhart Arnold's avatar Eckhart Arnold

- cstringview basic implementation (no optimazazions yet)

parent 87afe62f
"""cstringview.pyx - a cython-version of the stringview class for speedup
slicing strings without copying
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
StringView provides string-slicing without copying.
Slicing Python-strings always yields copies of a segment of the original
string. See: https://mail.python.org/pipermail/python-dev/2008-May/079699.html
However, this becomes costly (in terms of space and as a consequence also
time) when parsing longer documents. Unfortunately, Python's `memoryview`
does not work for unicode strings. Hence, the StringView class.
"""
import collections
from typing import Optional, Iterable, Tuple
__all__ = ('StringView', 'EMPTY_STRING_VIEW')
def pack_index(index, len):
index = index if index >= 0 else index + len
return 0 if index < 0 else len if index > len else index
def real_indices(begin, end, len):
if begin is None: begin = 0
if end is None: end = len
return pack_index(begin, len), pack_index(end, len)
class StringView(collections.abc.Sized):
""""
A rudimentary StringView class, just enough for the use cases
in parser.py. The difference between a StringView and the python
builtin strings is that StringView-objects do slicing without
copying, i.e. slices are just a view on a section of the sliced
string.
"""
__slots__ = ['text', 'begin', 'end', 'len', 'fullstring_flag']
def __init__(self, text: str, begin: Optional[int] = 0, end: Optional[int] = None) -> None:
self.text = text # type: str
self.begin = 0 # type: int
self.end = 0 # type: int
self.begin, self.end = real_indices(begin, end, len(text))
self.len = max(self.end - self.begin, 0)
self.fullstring_flag = (self.begin == 0 and self.len == len(self.text))
def __bool__(self):
return self.end > self.begin # and bool(self.text)
def __len__(self):
return self.len
def __str__(self):
if self.fullstring_flag: # optimization: avoid slicing/copying
return self.text
# since the slice is being copyied now, anyway, the copy might
# as well be stored in the string view
self.text = self.text[self.begin:self.end]
self.begin = 0
self.len = len(self.text)
self.end = self.len
self.fullstring_flag = True
return self.text
def __eq__(self, other):
return str(self) == str(other) # PERFORMANCE WARNING: This creates copies of the strings
def __hash__(self):
return hash(str(self)) # PERFORMANCE WARNING: This creates a copy of the string
def __add__(self, other):
if isinstance(other, str):
return (str(self) + other)
else:
return StringView(str(self) + str(other))
def __radd__(self, other):
if isinstance(other, str):
return (other + str(self))
else:
return StringView(str(other) + str(self))
def __getitem__(self, index):
# assert isinstance(index, slice), "As of now, StringView only allows slicing."
# assert index.step is None or index.step == 1, \
# "Step sizes other than 1 are not yet supported by StringView"
start, stop = real_indices(index.start, index.stop, self.len)
return StringView(self.text, self.begin + start, self.begin + stop)
def count(self, sub, start=None, end=None) -> int:
if self.fullstring_flag:
return self.text.count(sub, start, end)
elif start is None and end is None:
return self.text.count(sub, self.begin, self.end)
else:
start, end = real_indices(start, end, self.len)
return self.text.count(sub, self.begin + start, self.begin + end)
def find(self, sub, start=None, end=None) -> int:
if self.fullstring_flag:
return self.text.find(sub, start, end)
elif start is None and end is None:
return self.text.find(sub, self.begin, self.end) - self.begin
else:
start, end = real_indices(start, end, self.len)
return self.text.find(sub, self.begin + start, self.begin + end) - self.begin
def rfind(self, sub, start=None, end=None) -> int:
if self.fullstring_flag:
return self.text.rfind(sub, start, end)
if start is None and end is None:
return self.text.rfind(sub, self.begin, self.end) - self.begin
else:
start, end = real_indices(start, end, self.len)
return self.text.rfind(sub, self.begin + start, self.begin + end) - self.begin
def startswith(self, prefix: str, start: int = 0, end: Optional[int] = None) -> bool:
start += self.begin
end = self.end if end is None else self.begin + end
return self.text.startswith(prefix, start, end)
def match(self, regex):
return regex.match(self.text, pos=self.begin, endpos=self.end)
def index(self, absolute_index: int) -> int:
"""
Converts an index for a string watched by a StringView object
to an index relative to the string view object, e.g.:
>>> sv = StringView('xxIxx')[2:3]
>>> match = sv.match(re.compile('I'))
>>> match.end()
3
>>> sv.index(match.end())
1
"""
return absolute_index - self.begin
def indices(self, absolute_indices: Iterable[int]) -> Tuple[int, ...]:
"""Converts indices for a string watched by a StringView object
to indices relative to the string view object. See also: `sv_index()`
"""
return tuple(index - self.begin for index in absolute_indices)
def search(self, regex):
return regex.search(self.text, pos=self.begin, endpos=self.end)
def strip(self):
if self.fullstring_flag:
return self.text.strip()
else:
begin = self.begin
end = self.end
while begin < end and self.text[begin] in ' \n\t':
begin += 1
while end > begin and self.text[end] in ' \n\t':
end -= 1
return self.text[begin:end]
# return str(self).strip() # PERFORMANCE WARNING: This creates a copy of the string
def split(self, sep=None):
if self.fullstring_flag:
return self.text.split(sep)
else:
pieces = []
l = len(sep)
k = 0
i = self.find(sep, k)
while i >= 0:
pieces.append(self.text[self.begin + k : self.begin + i])
k = i + l
i = self.find(sep, k)
pieces.append(self.text[self.begin + k : self.end])
return pieces
# return str(self).split(sep, maxsplit) # PERFORMANCE WARNING: This creates a copy of the string
EMPTY_STRING_VIEW = StringView('')
......@@ -75,11 +75,15 @@ except ImportError:
from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union, Optional
from DHParser.toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, TransformationFunc, ParserBase, WHITESPACE_PTYPE, TOKEN_PTYPE, \
ZOMBIE_PARSER
from DHParser.error import Error, is_error, has_errors, linebreaks, line_col
from DHParser.toolkit import load_if_file
try:
import pyximport; pyximport.install()
from DHParser.cstringview import StringView, EMPTY_STRING_VIEW
except ImportError:
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
__all__ = ('PreprocessorFunc',
'HistoryRecord',
......
......@@ -33,8 +33,12 @@ except ImportError:
Iterator, Iterable, List, NamedTuple, Sequence, Union, Text, Tuple, Hashable
from DHParser.toolkit import is_logging, log_dir, identity
from DHParser.stringview import StringView
from DHParser.error import Error, linebreaks, line_col
try:
import pyximport; pyximport.install()
from DHParser.cstringview import StringView
except ImportError:
from DHParser.stringview import StringView
__all__ = ('ParserBase',
'WHITESPACE_PTYPE',
......@@ -224,7 +228,9 @@ class Node(collections.abc.Sized):
def __str__(self):
if self.children:
return "".join(str(child) for child in self.children)
return str(self.result)
elif isinstance(self.result, StringView):
self.result = str(self.result)
return self.result
def __repr__(self):
......@@ -277,8 +283,9 @@ class Node(collections.abc.Sized):
# or isinstance(result, str)), str(result)
# Possible optimization: Do not allow single nodes as argument:
# assert not isinstance(result, Node)
self._result = (result,) if isinstance(result, Node) else str(result) \
if isinstance(result, StringView) else result or '' # type: StrictResultType
self._result = (result,) if isinstance(result, Node) else result or '' # type: StrictResultType
# self._result = (result,) if isinstance(result, Node) else str(result) \
# if isinstance(result, StringView) else result or '' # type: StrictResultType
self.children = cast(ChildrenType, self._result) \
if isinstance(self._result, tuple) else cast(ChildrenType, ()) # type: ChildrenType
if self.children:
......
#from distutils.core import setup
from setuptools import setup
try:
from Cython.Build import cythonize
except ImportError:
def cythonize(filename):
return []
from DHParser.versionnumber import __version__
......@@ -10,6 +15,7 @@ setup(
name='DHParser',
version=__version__,
packages=['DHParser'],
ext_modules = cythonize('DHParser/cstringview.pyx')
url='https://gitlab.lrz.de/badw-it/DHParser',
license='MIT License (https://opensource.org/licenses/MIT)',
author='Eckhart Arnold',
......
#!/usr/bin/python3
"""test_stringview.py - tests of the stringview-module of DHParser
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2017 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
try:
import regex as re
except ImportError:
import re
sys.path.extend(['../', './'])
import pyximport; pyximport.install()
from DHParser.cstringview import StringView, EMPTY_STRING_VIEW, real_indices
class TestStringView:
def test_real_indices(self):
assert real_indices(3, 5, 10) == (3, 5)
assert real_indices(None, None, 10) == (0, 10)
assert real_indices(-2, -1, 10) == (8, 9)
assert real_indices(-3, 11, 10) == (7, 10)
assert real_indices(-5, -12, 10) == (5, 0)
assert real_indices(-12, -5, 10) == (0, 5)
assert real_indices(7, 6, 10) == (7, 6)
assert real_indices(None, 0, 10) == (0, 0)
def test_creation(self):
s = "0123456789"
assert str(StringView(s)) == s
assert str(StringView(s, 3, 4)) == '3'
assert str(StringView(s, -4)) == '6789'
def test_equality(self):
s = "0123456789"
assert StringView(s) == s
assert StringView(s, 3, 4) == '3'
assert StringView(s, -4) == '6789'
def test_slicing(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv == '0123456789'
assert sv[3:4] == '3'
assert sv[-3:-1] == '78'
assert sv[4:3] == ''
assert sv[:4] == '0123'
assert sv[4:] == '456789'
assert sv[-2:] == '89'
assert sv[:-5] == '01234'
assert isinstance(sv[3:5], StringView)
def test_len(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert len(sv) == 10
assert sv.len == 10
assert len(sv[5:5]) == 0
assert len(sv[7:4]) == 0
assert len(sv[-12:-2]) == 8
assert len(sv[-12:12]) == 10
def test_bool(self):
assert not StringView('')
assert StringView('x')
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert not sv[5:4]
assert sv[4:5], str(sv[4:5])
assert not sv[3:3]
assert not sv[12:13]
assert sv[0:20]
def test_sv_match(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.match(re.compile(r'\d'))
assert sv.match(re.compile(r'\d+'))
assert not sv.match(re.compile(r' '))
assert sv[4:].match(re.compile(r'45'))
def test_sv_search(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.search(re.compile(r'5'))
assert not sv.search(re.compile(r' '))
assert sv[5:].search(re.compile(r'5'))
assert not sv[:9].search(re.compile(r'9'))
def test_find(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.find('5') == 5
assert sv.find(' ') < 0
assert sv.find('0', 1) < 0
assert sv.find('9', 0, 8) < 0
assert sv.find('45', 1, 8) == 4
def test_startswith(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.startswith('012')
assert sv.startswith('123', 1)
assert not sv.startswith('123', 1, 3)
def test_EMPTY_STRING_VIEW(self):
assert len(EMPTY_STRING_VIEW) == 0
assert EMPTY_STRING_VIEW.find('x') < 0
assert not EMPTY_STRING_VIEW.match(re.compile(r'x'))
assert EMPTY_STRING_VIEW.match(re.compile(r'.*'))
assert len(EMPTY_STRING_VIEW[0:1]) == 0
if __name__ == "__main__":
from DHParser.testing import runner
runner("", globals())
......@@ -25,12 +25,16 @@ from functools import partial
sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging, logging, compile_python_object
from DHParser.stringview import StringView
from DHParser.error import Error
from DHParser.parser import compile_source, Retrieve, Grammar, Forward, Token, ZeroOrMore, RE, \
RegExp, Lookbehind, NegativeLookahead, OneOrMore, Series, Alternative
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import grammar_provider, DHPARSER_IMPORTS
try:
import pyximport; pyximport.install()
from DHParser.cstringview import StringView
except ImportError:
from DHParser.stringview import StringView
class TestInfiLoopsAndRecursion:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment