Author: Richard Plangger <planri...@gmail.com>
Branch: fix-gen-dfa
Changeset: r84575:fb294db3a207
Date: 2016-05-22 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/fb294db3a207/
Log: completed minimal test, refactored output to take the previously
generated states and reuse them. that is much easier than
reextracting the state from the string that is generated in DFA
diff --git a/pypy/interpreter/pyparser/genpytokenize.py
b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -191,7 +191,7 @@
newArcPair(states, EMPTY),
pseudoExtras, number, funny, contStr, name))
dfaStates, dfaAccepts = nfaToDfa(states, *pseudoToken)
- return DFA(dfaStates, dfaAccepts)
+ return DFA(dfaStates, dfaAccepts), dfaStates
# ______________________________________________________________________
@@ -205,7 +205,9 @@
newArcPair(states, DEFAULT),
any(states, notGroupStr(states, "'\\")))),
newArcPair(states, "'"))
- singleDFA = DFA(*nfaToDfa(states, *single))
+ states, accepts = nfaToDfa(states, *single)
+ singleDFA = DFA(states, accepts)
+ states_singleDFA = states
states = []
double = chain(states,
any(states, notGroupStr(states, '"\\')),
@@ -215,7 +217,9 @@
newArcPair(states, DEFAULT),
any(states, notGroupStr(states, '"\\')))),
newArcPair(states, '"'))
- doubleDFA = DFA(*nfaToDfa(states, *double))
+ states, accepts = nfaToDfa(states, *double)
+ doubleDFA = DFA(states, accepts)
+ states_doubleDFA = states
states = []
single3 = chain(states,
any(states, notGroupStr(states, "'\\")),
@@ -230,7 +234,9 @@
notChainStr(states, "''"))),
any(states, notGroupStr(states, "'\\")))),
chainStr(states, "'''"))
- single3DFA = NonGreedyDFA(*nfaToDfa(states, *single3))
+ states, accepts = nfaToDfa(states, *single3)
+ single3DFA = NonGreedyDFA(states, accepts)
+ states_single3DFA = states
states = []
double3 = chain(states,
any(states, notGroupStr(states, '"\\')),
@@ -245,9 +251,11 @@
notChainStr(states, '""'))),
any(states, notGroupStr(states, '"\\')))),
chainStr(states, '"""'))
- double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
- map = {"'" : singleDFA,
- '"' : doubleDFA,
+ states, accepts = nfaToDfa(states, *double3)
+ double3DFA = NonGreedyDFA(states, accepts)
+ states_double3DFA = states
+ map = {"'" : (singleDFA, states_singleDFA),
+ '"' : (doubleDFA, states_doubleDFA),
"r" : None,
"R" : None,
"u" : None,
@@ -257,13 +265,13 @@
for uniPrefix in ("", "u", "U", "b", "B", ):
for rawPrefix in ("", "r", "R"):
prefix = uniPrefix + rawPrefix
- map[prefix + "'''"] = single3DFA
- map[prefix + '"""'] = double3DFA
+ map[prefix + "'''"] = (single3DFA, states_single3DFA)
+ map[prefix + '"""'] = (double3DFA, states_doubleDFA)
return map
# ______________________________________________________________________
-def output(name, dfa_class, dfa):
+def output(name, dfa_class, dfa, states):
import textwrap
lines = []
i = 0
@@ -277,13 +285,13 @@
i += 1
import StringIO
lines.append("states = [\n")
- for numstate, state in enumerate(dfa.states):
+ for numstate, state in enumerate(states):
lines.append(" #")
lines.append(str(numstate))
lines.append('\n')
s = StringIO.StringIO()
i = 0
- for k, v in enumerate(state):
+ for k, v in sorted(state.items()):
i += 1
if k == '\x00default':
k = "automata.DEFAULT"
@@ -314,13 +322,17 @@
return ''.join(lines)
def main ():
- pseudoDFA = makePyPseudoDFA()
- print output("pseudoDFA", "DFA", pseudoDFA)
+ pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
+ print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
endDFAMap = makePyEndDFAMap()
- print output("double3DFA", "NonGreedyDFA", endDFAMap['"""'])
- print output("single3DFA", "NonGreedyDFA", endDFAMap["'''"])
- print output("singleDFA", "DFA", endDFAMap["'"])
- print output("doubleDFA", "DFA", endDFAMap['"'])
+ dfa, states = endDFAMap['"""']
+ print output("double3DFA", "NonGreedyDFA", dfa, states)
+ dfa, states = endDFAMap["'''"]
+ print output("single3DFA", "NonGreedyDFA", dfa, states)
+ dfa, states = endDFAMap["'"]
+ print output("singleDFA", "DFA", dfa, states)
+ dfa, states = endDFAMap["\""]
+ print output("doubleDFA", "DFA", dfa, states)
# ______________________________________________________________________
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py
b/pypy/interpreter/pyparser/test/test_gendfa.py
--- a/pypy/interpreter/pyparser/test/test_gendfa.py
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -2,10 +2,15 @@
from pypy.interpreter.pyparser.genpytokenize import output
def test_states():
- d = DFA([{"\x00": 1}, {"\x01": 0}], [False, True])
- assert output('test', DFA, d) == """\
+ states = [{"\x00": 1}, {"\x01": 0}]
+ d = DFA(states[:], [False, True])
+ assert output('test', DFA, d, states) == """\
accepts = [False, True]
states = [
+ #0
+ {'\\x00': 1},
+ #1
+ {'\\x01': 0},
]
test = automata.pypy.interpreter.pyparser.automata.DFA(states, accepts)
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
