Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
2d8f9c9f
Commit
2d8f9c9f
authored
Jul 10, 2018
by
di68kap
Browse files
- Transision to new Token-Model started
parent
08376d73
Changes
2
Hide whitespace changes
Inline
Side-by-side
DHParser/parse.py
View file @
2d8f9c9f
...
...
@@ -38,7 +38,7 @@ from DHParser.log import is_logging, HistoryRecord
from
DHParser.preprocess
import
BEGIN_TOKEN
,
END_TOKEN
,
RX_TOKEN_NAME
from
DHParser.stringview
import
StringView
,
EMPTY_STRING_VIEW
from
DHParser.syntaxtree
import
Node
,
RootNode
,
ParserBase
,
WHITESPACE_PTYPE
,
\
PLAINTEXT_PTYPE
,
TOKEN_PTYPE
,
ZOMBIE_PARSER
TOKEN_PTYPE
,
ZOMBIE_PARSER
from
DHParser.toolkit
import
sane_parser_name
,
escape_control_characters
,
re
,
typing
from
typing
import
Callable
,
cast
,
Dict
,
DefaultDict
,
List
,
Set
,
Tuple
,
Union
,
Optional
...
...
@@ -260,7 +260,7 @@ class Parser(ParserBase):
"""
duplicate
=
self
.
__class__
()
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
reset
(
self
):
...
...
@@ -864,7 +864,7 @@ class PreprocessorToken(Parser):
def
__deepcopy__
(
self
,
memo
):
duplicate
=
self
.
__class__
(
self
.
name
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -894,27 +894,28 @@ class PreprocessorToken(Parser):
return
None
,
text
class
PlainText
(
Parser
):
class
Token
(
Parser
):
"""
Parses plain text strings. (Could be done by RegExp as well, but is faster.)
Example::
>>> while_token =
PlainText
("while")
>>> while_token =
Token
("while")
>>> Grammar(while_token)("while").content
'while'
"""
assert
PLAINTEXT
_PTYPE
==
":
PlainText
"
assert
TOKEN
_PTYPE
==
":
_Token
"
def
__init__
(
self
,
text
:
str
)
->
None
:
super
().
__init__
()
self
.
text
=
text
self
.
len
=
len
(
text
)
self
.
ptype
=
":_Token"
# TODO: Remove when transition to New Token class is finished
def
__deepcopy__
(
self
,
memo
):
duplicate
=
self
.
__class__
(
self
.
text
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype =
self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -958,7 +959,7 @@ class RegExp(Parser):
regexp
=
self
.
regexp
.
pattern
duplicate
=
self
.
__class__
(
regexp
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -985,6 +986,7 @@ class Whitespace(RegExp):
assert
WHITESPACE_PTYPE
==
":Whitespace"
# def RE(regexp, wL=None, wR=None) -> 'Series':
# def rxp(regex):
# return regex if isinstance(regex, RegExp) else RegExp(regex)
...
...
@@ -1083,7 +1085,7 @@ class _RE(Parser):
regexp
=
self
.
main
.
regexp
.
pattern
duplicate
=
self
.
__class__
(
regexp
,
self
.
rx_wsl
,
self
.
rx_wsr
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -1146,14 +1148,14 @@ class _Token(_RE):
def
__deepcopy__
(
self
,
memo
=
{}):
duplicate
=
self
.
__class__
(
self
.
token
,
self
.
rx_wsl
,
self
.
rx_wsr
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__repr__
(
self
):
return
'"%s"'
%
self
.
token
if
self
.
token
.
find
(
'"'
)
<
0
else
"'%s'"
%
self
.
token
def
create_main_parser
(
self
,
arg
)
->
Parser
:
return
PlainText
(
arg
)
return
Token
(
arg
)
########################################################################
...
...
@@ -1184,7 +1186,7 @@ class UnaryOperator(Parser):
parser
=
copy
.
deepcopy
(
self
.
parser
,
memo
)
duplicate
=
self
.
__class__
(
parser
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
apply
(
self
,
func
:
Parser
.
ApplyFunc
)
->
bool
:
...
...
@@ -1215,7 +1217,7 @@ class NaryOperator(Parser):
parsers
=
copy
.
deepcopy
(
self
.
parsers
,
memo
)
duplicate
=
self
.
__class__
(
*
parsers
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
apply
(
self
,
func
:
Parser
.
ApplyFunc
)
->
bool
:
...
...
@@ -1400,7 +1402,7 @@ class Series(NaryOperator):
parsers
=
copy
.
deepcopy
(
self
.
parsers
,
memo
)
duplicate
=
self
.
__class__
(
*
parsers
,
mandatory
=
self
.
mandatory
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -1846,7 +1848,7 @@ class Retrieve(Parser):
def
__deepcopy__
(
self
,
memo
):
duplicate
=
self
.
__class__
(
self
.
symbol
,
self
.
filter
)
duplicate
.
name
=
self
.
name
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
return
duplicate
def
__call__
(
self
,
text
:
StringView
)
->
Tuple
[
Optional
[
Node
],
StringView
]:
...
...
@@ -1968,7 +1970,7 @@ class Forward(Parser):
def
__deepcopy__
(
self
,
memo
):
duplicate
=
self
.
__class__
()
# duplicate.name = self.name # Forward-Parsers should never have a name!
#
duplicate.ptype = self.ptype
duplicate
.
ptype
=
self
.
ptype
memo
[
id
(
self
)]
=
duplicate
parser
=
copy
.
deepcopy
(
self
.
parser
,
memo
)
duplicate
.
set
(
parser
)
...
...
DHParser/syntaxtree.py
View file @
2d8f9c9f
...
...
@@ -35,7 +35,6 @@ from typing import Callable, cast, Iterator, List, AbstractSet, Set, Union, Tupl
__all__
=
(
'ParserBase'
,
'WHITESPACE_PTYPE'
,
'PLAINTEXT_PTYPE'
,
'TOKEN_PTYPE'
,
'MockParser'
,
'ZombieParser'
,
...
...
@@ -111,7 +110,6 @@ class ParserBase:
WHITESPACE_PTYPE
=
':Whitespace'
PLAINTEXT_PTYPE
=
':PlainText'
TOKEN_PTYPE
=
':_Token'
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment