branch: externals/parser-generator
commit e444779776cdf15b846ae61a6a0d2df0e091d625
Author: Christian Johansson <[email protected]>
Commit: Christian Johansson <[email protected]>
Implemented fix for export lambdas not passing byte compilation
---
docs/Lexical-Analysis.md | 14 +++--
docs/Syntax-Analysis/LL1.md | 12 ++---
docs/Syntax-Analysis/LLk.md | 12 ++---
docs/Syntax-Analysis/LR0.md | 21 ++++++++
docs/Syntax-Analysis/LRk.md | 18 +++++++
parser-generator-lex-analyzer.el | 10 ++++
parser-generator-ll-export.el | 18 +++----
parser-generator-lr-export.el | 18 +++----
test/parser-generator-ll-export-test.el | 49 ++++++++++++++++++
test/parser-generator-lr-export-test.el | 91 ++++++++++++++++++++++++++++++++-
10 files changed, 224 insertions(+), 39 deletions(-)
diff --git a/docs/Lexical-Analysis.md b/docs/Lexical-Analysis.md
index c29e907844..dab8c2db54 100644
--- a/docs/Lexical-Analysis.md
+++ b/docs/Lexical-Analysis.md
@@ -6,13 +6,17 @@ The lexical analysis is internally indexed on a local
variable `parser-generator
All parsers expect a list as response from the lexical-analysis, the first
item in the list should be a list of one or more tokens. The second is "move
index"-flag, if it is non-nil it is expected to be a integer representing the
index to temporarily move the index to and perform a new lex at. The third item
is the new state after the lex. Return values 2 and 3 are optional.
-To enable exporting, the functions need to be specified in a way that the
entire body is within the same block, do that using `(let)` or `(progn)` for
example.
+To enable exporting, the variables
`'parser-generator-lex-analyzer--function-export-string` and
`'parser-generator-lex-analyzer--get-function-export-string` need to declared
as string representations of their original source code, this is because Emacs
compilation obfuscates lambda source code at the some time in compilation.
```emacs-lisp
(setq
- parser-generator-lex-analyzer--function
- (lambda (index _state)
- (let* ((string '(("a" 1 . 2) ("a" 2 . 3) ("b" 3 . 4) ("b" 4 . 5)))
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (let* ((string '((\"a\" 1 . 2) (\"a\" 2 . 3) (\"b\" 3 . 4) (\"b\" 4 . 5)))
(string-length (length string))
(max-index index)
(tokens))
@@ -21,7 +25,7 @@ To enable exporting, the functions need to be specified in a
way that the entire
(< (1- index) max-index))
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
- (list tokens))))
+ (list tokens)))")
```
## Token
diff --git a/docs/Syntax-Analysis/LL1.md b/docs/Syntax-Analysis/LL1.md
index 0d0e5d9db2..d24d83cf3e 100644
--- a/docs/Syntax-Analysis/LL1.md
+++ b/docs/Syntax-Analysis/LL1.md
@@ -135,8 +135,8 @@ Each production RHS can optionally contain a
lambda-expression that will be call
(parser-generator-process-grammar)
(parser-generator-ll-generate-table)
(setq
- parser-generator-lex-analyzer--function
- (lambda (index)
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index)
(let* ((string '((a 1 . 2) (b 2 . 3) (b 3 . 4) (a 4 . 5) (b 5 . 6)))
(string-length (length string))
(max-index index)
@@ -146,11 +146,11 @@ Each production RHS can optionally contain a
lambda-expression that will be call
(< (1- index) max-index))
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
- (nreverse tokens))))
+ (nreverse tokens)))")
(setq
- parser-generator-lex-analyzer--get-function
- (lambda (token)
- (car token)))
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
(let ((export (parser-generator-ll-export-to-elisp "ba3")))
(with-temp-buffer
(insert export)
diff --git a/docs/Syntax-Analysis/LLk.md b/docs/Syntax-Analysis/LLk.md
index bf7130b452..af1b15401f 100644
--- a/docs/Syntax-Analysis/LLk.md
+++ b/docs/Syntax-Analysis/LLk.md
@@ -133,8 +133,8 @@ Each production RHS can optionally contain a
lambda-expression that will be call
(parser-generator-process-grammar)
(parser-generator-ll-generate-table)
(setq
- parser-generator-lex-analyzer--function
- (lambda (index)
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index)
(let* ((string '((b 1 . 2) (b 2 . 3) (a 3 . 4)))
(string-length (length string))
(max-index index)
@@ -144,11 +144,11 @@ Each production RHS can optionally contain a
lambda-expression that will be call
(< (1- index) max-index))
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
- (nreverse tokens))))
+ (nreverse tokens)))")
(setq
- parser-generator-lex-analyzer--get-function
- (lambda (token)
- (car token)))
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
(let ((export (parser-generator-ll-export-to-elisp "ba")))
(with-temp-buffer
(insert export)
diff --git a/docs/Syntax-Analysis/LR0.md b/docs/Syntax-Analysis/LR0.md
index e6df7bc630..1773e5b184 100644
--- a/docs/Syntax-Analysis/LR0.md
+++ b/docs/Syntax-Analysis/LR0.md
@@ -230,6 +230,27 @@ The export should be executed after a parser has been
generated, example:
'(5 3 5 2)
(parser-generator-lr-parse)))
+ ;; Setup lex-analyzer
+ (setq
+ parser-generator-lex-analyzer--function-export-string-export-string
+ "(lambda (index)
+ (with-current-buffer buffer
+ (when (<= (+ index 1) (point-max))
+ (let ((start index)
+ (end (+ index 1)))
+ (let ((token (buffer-substring-no-properties start end)))
+ `(,token ,start . ,end))))))")
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (with-current-buffer buffer
+ (let ((start (car (cdr token)))
+ (end (cdr (cdr token))))
+ (when (<= end (point-max))
+ (buffer-substring-no-properties
+ start
+ end)))))")
+
;; Export parser
(let ((export (parser-generator-lr-export-to-elisp "e--")))
diff --git a/docs/Syntax-Analysis/LRk.md b/docs/Syntax-Analysis/LRk.md
index 8cbeb51df6..7fb9c663a2 100644
--- a/docs/Syntax-Analysis/LRk.md
+++ b/docs/Syntax-Analysis/LRk.md
@@ -275,6 +275,24 @@ The export should be executed after a parser has been
generated, example:
'(2 2 2 1 1)
(parser-generator-lr-parse)))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index)
+ (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5)))
+ (string-length (length string))
+ (max-index index)
+ (tokens))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (push (nth (1- index) string) tokens)
+ (setq index (1+ index)))
+ (nreverse tokens)))")
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
+
;; Export parser
(let ((export (parser-generator-lr-export-to-elisp "e--")))
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index 2e9a48d109..08e7755578 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -20,11 +20,21 @@
nil
"Get token contents. Any return is valid even nil.")
+(defvar
+ parser-generator-lex-analyzer--get-function-export-string
+ ""
+ "String version of get function, used only when generating exports. This is
needed because Emacs compilation obfuscates lambda source code after so it is
not printable anymore. Easiest way to populate this is to copy the lambda
source code into a string and manually escape double quotes.")
+
(defvar
parser-generator-lex-analyzer--function
nil
"Get next token like \='(a b . c) or nil, expects signal if input-tape is
invalid.")
+(defvar
+ parser-generator-lex-analyzer--function-export-string
+ ""
+ "String version of get function, used only when generating exports. This is
needed because Emacs compilation obfuscates lambda source code after so it is
not printable anymore. Easiest way to populate this is to copy the lambda
source code into a string and manually escape double quotes.")
+
(defvar
parser-generator-lex-analyzer--state-init
nil
diff --git a/parser-generator-ll-export.el b/parser-generator-ll-export.el
index 310dcf38ab..e5fefafeab 100644
--- a/parser-generator-ll-export.el
+++ b/parser-generator-ll-export.el
@@ -34,10 +34,10 @@
(error "Table for terminals is undefined!"))
(unless parser-generator--table-translations
(error "Table for translations by production-number is undefined!"))
- (unless parser-generator-lex-analyzer--get-function
- (error "Missing lex-analyzer get function!"))
- (unless parser-generator-lex-analyzer--function
- (error "Missing lex-analyzer function!"))
+ (unless parser-generator-lex-analyzer--get-function-export-string
+ (error "Missing lex-analyzer get function exporting string!"))
+ (unless parser-generator-lex-analyzer--function-export-string
+ (error "Missing lex-analyzer function export string!"))
(let ((code))
(with-temp-buffer
@@ -165,18 +165,16 @@
;; Lex-Analyzer Get Function
(insert
(format
- "(defvar\n %s-lex-analyzer--get-function\n (lambda %S %S)\n \"The
lex-analyzer get function.\")\n\n"
+ "(defvar\n %s-lex-analyzer--get-function\n %s\n \"The lex-analyzer
get function.\")\n\n"
namespace
- (aref parser-generator-lex-analyzer--get-function 0)
- (car (aref parser-generator-lex-analyzer--get-function 1))))
+ parser-generator-lex-analyzer--get-function-export-string))
;; Lex-Analyzer Function
(insert
(format
- "(defvar\n %s-lex-analyzer--function\n (lambda %S %S)\n \"The
lex-analyzer function.\")\n\n"
+ "(defvar\n %s-lex-analyzer--function\n %s\n \"The lex-analyzer
function.\")\n\n"
namespace
- (aref parser-generator-lex-analyzer--function 0)
- (car (aref parser-generator-lex-analyzer--function 1))))
+ parser-generator-lex-analyzer--function-export-string))
;; Lex-Analyzer Reset Function
(insert
diff --git a/parser-generator-lr-export.el b/parser-generator-lr-export.el
index aeb41838d3..657ec7d8cd 100644
--- a/parser-generator-lr-export.el
+++ b/parser-generator-lr-export.el
@@ -40,10 +40,10 @@
(error "Table for terminals is undefined!"))
(unless parser-generator--table-translations
(error "Table for translations by production-number is undefined!"))
- (unless parser-generator-lex-analyzer--get-function
- (error "Missing lex-analyzer get function!"))
- (unless parser-generator-lex-analyzer--function
- (error "Missing lex-analyzer function!"))
+ (unless parser-generator-lex-analyzer--get-function-export-string
+ (error "Missing lex-analyzer get function export string!"))
+ (unless parser-generator-lex-analyzer--function-export-string
+ (error "Missing lex-analyzer function export string!"))
(let ((code))
(with-temp-buffer
@@ -186,18 +186,16 @@
;; Lex-Analyzer Get Function
(insert
(format
- "(defvar\n %s-lex-analyzer--get-function\n (lambda %S %S)\n \"The
lex-analyzer get function.\")\n\n"
+ "(defvar\n %s-lex-analyzer--get-function\n %s\n \"The lex-analyzer
get function.\")\n\n"
namespace
- (aref parser-generator-lex-analyzer--get-function 0)
- (car (aref parser-generator-lex-analyzer--get-function 1))))
+ parser-generator-lex-analyzer--get-function-export-string))
;; Lex-Analyzer Function
(insert
(format
- "(defvar\n %s-lex-analyzer--function\n (lambda %S %S)\n \"The
lex-analyzer function.\")\n\n"
+ "(defvar\n %s-lex-analyzer--function\n %s\n \"The lex-analyzer
function.\")\n\n"
namespace
- (aref parser-generator-lex-analyzer--function 0)
- (car (aref parser-generator-lex-analyzer--function 1))))
+ parser-generator-lex-analyzer--function-export-string))
;; Lex-Analyzer Reset Function
(insert
diff --git a/test/parser-generator-ll-export-test.el
b/test/parser-generator-ll-export-test.el
index 41bab82325..c663066cfb 100644
--- a/test/parser-generator-ll-export-test.el
+++ b/test/parser-generator-ll-export-test.el
@@ -52,11 +52,29 @@
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
(list (nreverse tokens) nil index nil))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (let* ((string '((b 1 . 2) (b 2 . 3) (a 3 . 4)))
+ (string-length (length string))
+ (max-index index)
+ (tokens))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (push (nth (1- index) string) tokens)
+ (setq index (1+ index)))
+ (list (nreverse tokens) nil index nil)))")
(setq
parser-generator-lex-analyzer--get-function
(lambda (token)
(car token)))
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
+
(let ((export (parser-generator-ll-export-to-elisp "ba")))
(with-temp-buffer
(insert export)
@@ -95,6 +113,19 @@
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
(list (nreverse tokens) nil index nil))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (let* ((string '((b 1 . 2) (b 2 . 3) (b 3 . 4) (a 4 . 5)))
+ (string-length (length string))
+ (max-index index)
+ (tokens))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (push (nth (1- index) string) tokens)
+ (setq index (1+ index)))
+ (list (nreverse tokens) nil index nil)))")
(let ((export (parser-generator-ll-export-to-elisp "ba2")))
(with-temp-buffer
@@ -145,11 +176,29 @@
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
(list (nreverse tokens) nil index nil))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (let* ((string '((a 1 . 2) (b 2 . 3) (b 3 . 4) (a 4 . 5) (b 5 . 6)))
+ (string-length (length string))
+ (max-index index)
+ (tokens))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (push (nth (1- index) string) tokens)
+ (setq index (1+ index)))
+ (list (nreverse tokens) nil index nil)))")
(setq
parser-generator-lex-analyzer--get-function
(lambda (token)
(car token)))
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
+
(let ((export (parser-generator-ll-export-to-elisp "ba3")))
(with-temp-buffer
(insert export)
diff --git a/test/parser-generator-lr-export-test.el
b/test/parser-generator-lr-export-test.el
index f1d2b629c0..2e36b1fd0b 100644
--- a/test/parser-generator-lr-export-test.el
+++ b/test/parser-generator-lr-export-test.el
@@ -45,7 +45,6 @@
(end (+ index 1)))
(let ((token (buffer-substring-no-properties start end)))
(list `(,token ,start . ,end) nil end nil)))))))
-
(setq
parser-generator-lex-analyzer--get-function
(lambda (token)
@@ -64,6 +63,24 @@
"bbaa"
(parser-generator-lr-translate)))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (with-current-buffer \"*a*\"
+ (when (<= (+ index 1) (point-max))
+ (let ((start index)
+ (end (+ index 1)))
+ (let ((token (buffer-substring-no-properties start end)))
+ (list `(,token ,start . ,end) nil end nil))))))")
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (with-current-buffer \"*a*\"
+ (let ((start (car (cdr token)))
+ (end (cdr (cdr token))))
+ (when (<= end (point-max))
+ (buffer-substring-no-properties start end)))))")
+
;; Export parser
(let ((export (parser-generator-lr-export-to-elisp "ba")))
(parser-generator--debug
@@ -179,11 +196,28 @@
(push (nth (1- index) string) tokens)
(setq index (1+ index)))
(list (nreverse tokens) nil index nil))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5)))
+ (string-length (length string))
+ (max-index index)
+ (tokens))
+ (while (and
+ (< (1- index) string-length)
+ (< (1- index) max-index))
+ (push (nth (1- index) string) tokens)
+ (setq index (1+ index)))
+ (list (nreverse tokens) nil index nil)))")
(setq
parser-generator-lex-analyzer--get-function
(lambda (token)
(car token)))
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (car token))")
;; Test parse
(should
@@ -239,6 +273,15 @@
(end (+ index 1)))
(let ((token (buffer-substring-no-properties start end)))
(list `(,token ,start . ,end) nil end nil)))))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (with-current-buffer \"*a*\"
+ (when (<= (+ index 1) (point-max))
+ (let ((start index)
+ (end (+ index 1)))
+ (let ((token (buffer-substring-no-properties start end)))
+ (list `(,token ,start . ,end) nil end nil))))))")
(setq
parser-generator-lex-analyzer--get-function
@@ -248,6 +291,14 @@
(end (cdr (cdr token))))
(when (<= end (point-max))
(buffer-substring-no-properties start end))))))
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (with-current-buffer \"*a*\"
+ (let ((start (car (cdr token)))
+ (end (cdr (cdr token))))
+ (when (<= end (point-max))
+ (buffer-substring-no-properties start end)))))")
(should
(equal
@@ -311,6 +362,15 @@
(end (+ index 1)))
(let ((token (buffer-substring-no-properties start end)))
(list `(,token ,start . ,end) nil end nil)))))))
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (with-current-buffer \"*a*\"
+ (when (<= (+ index 1) (point-max))
+ (let ((start index)
+ (end (+ index 1)))
+ (let ((token (buffer-substring-no-properties start end)))
+ (list `(,token ,start . ,end) nil end nil))))))")
(setq
parser-generator-lex-analyzer--get-function
@@ -322,6 +382,16 @@
(buffer-substring-no-properties
start
end))))))
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (with-current-buffer \"*a*\"
+ (let ((start (car (cdr token)))
+ (end (cdr (cdr token))))
+ (when (<= end (point-max))
+ (buffer-substring-no-properties
+ start
+ end)))))")
(should
(equal
@@ -383,7 +453,6 @@
(end (+ index 1)))
(let ((token (buffer-substring-no-properties start end)))
(list `(,token ,start . ,end) nil end nil)))))))
-
(setq
parser-generator-lex-analyzer--get-function
(lambda (token)
@@ -400,6 +469,24 @@
(message "Passed translate before export")
+ (setq
+ parser-generator-lex-analyzer--function-export-string
+ "(lambda (index _state)
+ (with-current-buffer \"*a*\"
+ (when (<= (+ index 1) (point-max))
+ (let ((start index)
+ (end (+ index 1)))
+ (let ((token (buffer-substring-no-properties start end)))
+ (list `(,token ,start . ,end) nil end nil))))))")
+ (setq
+ parser-generator-lex-analyzer--get-function-export-string
+ "(lambda (token)
+ (with-current-buffer \"*a*\"
+ (let ((start (car (cdr token)))
+ (end (cdr (cdr token))))
+ (when (<= end (point-max))
+ (buffer-substring-no-properties start end)))))")
+
;; Export parser
(let ((export (parser-generator-lr-export-to-elisp "fa")))
(with-temp-buffer
