branch: externals/parser-generator
commit fa7089ef2dbdcd7eb199d4740e2a02d2f5a3eda8
Author: Christian Johansson <[email protected]>
Commit: Christian Johansson <[email protected]>
Re-factored lex analyzer function to not use length argument
---
parser-generator-lex-analyzer.el | 41 ++++++++++++++++++++++--------
parser-generator-lr.el | 10 +-------
test/parser-generator-lex-analyzer-test.el | 11 ++++----
test/parser-generator-lr-test.el | 16 ++++++------
4 files changed, 45 insertions(+), 33 deletions(-)
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index b7f6274..307d497 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -37,10 +37,27 @@
(error "Missing lex-analyzer function!"))
(unless parser-generator--look-ahead-number
(error "Missing look-ahead number!"))
- (funcall
- parser-generator-lex-analyzer--function
- parser-generator-lex-analyzer--index
- parser-generator--look-ahead-number))
+ (let ((look-ahead)
+ (look-ahead-length 0)
+ (index parser-generator-lex-analyzer--index))
+ (while (< look-ahead-length parser-generator--look-ahead-number)
+ (let ((next-look-ahead
+ (funcall
+ parser-generator-lex-analyzer--function
+ index)))
+ (if next-look-ahead
+ (progn
+ (unless (listp (car next-look-ahead))
+ (setq next-look-ahead (list next-look-ahead)))
+ (dolist (next-look-ahead-item next-look-ahead)
+ (when (< look-ahead-length parser-generator--look-ahead-number)
+ (push next-look-ahead-item look-ahead)
+ (setq look-ahead-length (1+ look-ahead-length))
+ (setq index (1+ index)))))
+ (push (list parser-generator--e-identifier) look-ahead)
+ (setq look-ahead-length (1+ look-ahead-length))
+ (setq index (1+ index)))))
+ (nreverse look-ahead)))
(defun parser-generator-lex-analyzer--pop-token ()
"Pop next token via lex-analyzer."
@@ -48,13 +65,15 @@
(error "Missing lex-analyzer index!"))
(unless parser-generator-lex-analyzer--function
(error "Missing lex-analyzer function!"))
- (let ((token (car (funcall
- parser-generator-lex-analyzer--function
- parser-generator-lex-analyzer--index
- 1))))
- (setq parser-generator-lex-analyzer--index
- (1+ parser-generator-lex-analyzer--index))
- token))
+ (let ((token (funcall
+ parser-generator-lex-analyzer--function
+ parser-generator-lex-analyzer--index)))
+ (unless (listp (car token))
+ (setq token (list token)))
+ (let ((first-token (car token)))
+ (setq parser-generator-lex-analyzer--index
+ (1+ parser-generator-lex-analyzer--index))
+ first-token)))
(defun parser-generator-lex-analyzer--reset ()
"Reset lex-analyzer."
diff --git a/parser-generator-lr.el b/parser-generator-lr.el
index 09fd2c4..b1f55f3 100644
--- a/parser-generator-lr.el
+++ b/parser-generator-lr.el
@@ -514,19 +514,11 @@
;; (1) The lookahead string u, consisting of the next k input symbols,
is determined.
(let ((look-ahead (parser-generator-lex-analyzer--peek-next-look-ahead))
- (look-ahead-length 0)
(look-ahead-full))
-
- (setq look-ahead-length (length look-ahead))
-
- ;; If we reached end of input-tape and look-ahead is too small, append
e-identifiers
- (while (< look-ahead-length parser-generator--look-ahead-number)
- (push parser-generator--e-identifier look-ahead)
- (setq look-ahead-length (1+ look-ahead-length)))
-
;; Save token stream indexes in separate variable if needed later
(setq look-ahead-full (nreverse look-ahead))
+ ;; Create simplified look-ahead for logic below
(setq look-ahead nil)
(dolist (look-ahead-item look-ahead-full)
(if (listp look-ahead-item)
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index 03d4a05..ad4df4c 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -22,10 +22,10 @@
(parser-generator-lex-analyzer--peek-next-look-ahead))
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
@@ -54,7 +54,7 @@
(setq parser-generator--look-ahead-number 10)
(should
(equal
- '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5))
+ '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) (e) (e) (e) (e) (e) (e))
(parser-generator-lex-analyzer--peek-next-look-ahead)))
(message "Ended tests for
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -71,10 +71,10 @@
(parser-generator-lex-analyzer--pop-token))
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '(("a" 1 . 2) ("b" 2 . 3)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
@@ -103,6 +103,7 @@
(defun parser-generator-lex-analyzer-test ()
"Run test."
+ ;; (setq debug-on-error t)
(parser-generator-lex-analyzer-test--peek-next-look-ahead)
(parser-generator-lex-analyzer-test--pop-token))
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 71017fa..82e9d3a 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -266,10 +266,10 @@
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
@@ -285,10 +285,10 @@
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5) (b 5 . 6)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
@@ -309,10 +309,10 @@
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '(("a" 1 . 2) ("a" 2 . 3) ("b" 3 . 4) ("b" 4 . 5)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
@@ -328,10 +328,10 @@
(setq
parser-generator-lex-analyzer--function
- (lambda (index length)
+ (lambda (index)
(let* ((string '(("a" 1 . 2) ("a" 2 . 3) ("b" 3 . 4) ("b" 4 . 5) ("b" 5 .
6)))
(string-length (length string))
- (max-index (+ index length))
+ (max-index (1+ index))
(tokens))
(while (and
(< index string-length)
