CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc
Committed By: agc
Date: Thu Jun 22 23:16:47 UTC 2023
Modified Files:
othersrc/external/bsd/elex/dist: agcre.c elex.c elex.h main.c
othersrc/external/bsd/elex/dist/tests: 28.expected
Log Message:
elex-20230622
+ restore bug fix lost in previous - reset yyleng to 0 when deleting input
from the parser
+ API change to make function name more descriptive in API
+ bring license up to date
+ bump version number for header file
To generate a diff of this commit:
cvs rdiff -u -r1.4 -r1.5 othersrc/external/bsd/elex/dist/agcre.c \
othersrc/external/bsd/elex/dist/elex.c \
othersrc/external/bsd/elex/dist/elex.h \
othersrc/external/bsd/elex/dist/main.c
cvs rdiff -u -r1.4 -r1.5 othersrc/external/bsd/elex/dist/tests/28.expected
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/dist/agcre.c
diff -u othersrc/external/bsd/elex/dist/agcre.c:1.4 othersrc/external/bsd/elex/dist/agcre.c:1.5
--- othersrc/external/bsd/elex/dist/agcre.c:1.4 Wed Jun 21 23:36:17 2023
+++ othersrc/external/bsd/elex/dist/agcre.c Thu Jun 22 23:16:46 2023
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013,2017 Alistair Crooks. All Rights reserved.
+ * Copyright (c) 2013,2017,2023 Alistair Crooks. All Rights reserved.
* All rights reserved.
*
* Parts of this are:
Index: othersrc/external/bsd/elex/dist/elex.c
diff -u othersrc/external/bsd/elex/dist/elex.c:1.4 othersrc/external/bsd/elex/dist/elex.c:1.5
--- othersrc/external/bsd/elex/dist/elex.c:1.4 Wed Jun 21 23:36:17 2023
+++ othersrc/external/bsd/elex/dist/elex.c Thu Jun 22 23:16:46 2023
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2018,2021 Alistair Crooks
+ * Copyright (c) 2018,2021,2023 Alistair Crooks
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -252,6 +252,7 @@ static int
delete_input(elex_t *elex, size_t n)
{
memmove(&elex->s[elex->cc], &elex->s[elex->cc + n], elex->len - elex->cc - n);
+ elex->yyleng = 0;
elex->len -= n;
elex->s[elex->len] = 0x0;
return 1;
@@ -696,7 +697,7 @@ elex_exec(elex_t *elex, const char *info
/* one function to access string values */
void *
-elex_exec_str(elex_t *elex, const char *info, uint64_t n, uint64_t *size)
+elex_exec_mem(elex_t *elex, const char *info, uint64_t n, uint64_t *size)
{
uint64_t len;
Index: othersrc/external/bsd/elex/dist/elex.h
diff -u othersrc/external/bsd/elex/dist/elex.h:1.4 othersrc/external/bsd/elex/dist/elex.h:1.5
--- othersrc/external/bsd/elex/dist/elex.h:1.4 Wed Jun 21 23:36:17 2023
+++ othersrc/external/bsd/elex/dist/elex.h Thu Jun 22 23:16:46 2023
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2018,2021 Alistair Crooks
+ * Copyright (c) 2018,2021,2023 Alistair Crooks
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -23,7 +23,7 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef ELEX_H_
-#define ELEX_H_ 20230621
+#define ELEX_H_ 20230622
#include
@@ -34,7 +34,7 @@
#define elex_new ELEX_NAMESPACE(LIB_NAMESPACE, elex_new)
#define elex_dispose ELEX_NAMESPACE(LIB_NAMESPACE, elex_dispose)
#define elex_exec ELEX_NAMESPACE(LIB_NAMESPACE, elex_exec)
-#define elex_exec_str ELEX_NAMESPACE(LIB_NAMESPACE, elex_exec_str)
+#define elex_exec_mem ELEX_NAMESPACE(LIB_NAMESPACE, elex_exec_mem)
#define elex_make_new_rule ELEX_NAMESPACE(LIB_NAMESPACE, elex_make_new_rule)
#endif
@@ -59,7 +59,7 @@ int elex_dispose(elex_t **/*elex*/);
/* these functions do ALL the work */
int64_t elex_exec(elex_t */*elex*/, const char */*info*/, uint64_t /*num*/, const char */*s*/, int64_t /*cc*/);
-void *elex_exec_str(elex_t */*elex*/, const char */*info*/, uint64_t /*n*/, uint64_t */*size*/);
+void *elex_exec_mem(elex_t */*elex*/, const char */*info*/, uint64_t /*n*/, uint64_t */*size*/);
/* with one exeception - deal with states */
int elex_make_new_rule(elex_t */*elex*/, const char */*startstate*/,
Index: othersrc/external/bsd/elex/dist/main.c
diff -u othersrc/external/bsd/elex/dist/main.c:1.4 othersrc/external/bsd/elex/dist/main.c:1.5
--- othersrc/external/bsd/elex/dist/main.c:1.4 Wed Jun 21 23:36:17 2023
+++ othersrc/external/bsd/elex/dist/main.c Thu Jun 22 23:16:46 2023
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2018,2021 Alistair Crooks
+ * Copyright (c) 2018,2021,2023 Alistair Crooks
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -138,7 +138,7 @@ main(int argc, char **argv)
}
while (elex_exec(elex, "next-token", 0, NULL, 0) != 0) {
type = elex_exec(elex, "get-yytype", 0, NULL, 0);
- text = elex_exec_str(elex, "get-yytext", 0, &len);
+ text = elex_exec_mem(elex, "get-yytext", 0, &len);
if (graphic) {
switch(type) {
case /* "IDENT" */ 0xdb8ea4d:
Index: othersrc/external/bsd/elex/dist/tests/28.expected
diff -u othersrc/external/bsd/elex/dist/te
CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc Committed By: agc Date: Thu Jun 22 23:16:47 UTC 2023 Modified Files: othersrc/external/bsd/elex/dist: agcre.c elex.c elex.h main.c othersrc/external/bsd/elex/dist/tests: 28.expected Log Message: elex-20230622 + restore bug fix lost in previous - reset yyleng to 0 when deleting input from the parser + API change to make function name more descriptive in API + bring license up to date + bump version number for header file To generate a diff of this commit: cvs rdiff -u -r1.4 -r1.5 othersrc/external/bsd/elex/dist/agcre.c \ othersrc/external/bsd/elex/dist/elex.c \ othersrc/external/bsd/elex/dist/elex.h \ othersrc/external/bsd/elex/dist/main.c cvs rdiff -u -r1.4 -r1.5 othersrc/external/bsd/elex/dist/tests/28.expected Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Wed Jun 21 23:36:17 UTC 2023
Modified Files:
othersrc/external/bsd/elex: TODO
othersrc/external/bsd/elex/dist: Makefile agcre.c agcre.h elex.c elex.h
main.c
othersrc/external/bsd/elex/dist/tests: 28.expected
othersrc/external/bsd/elex/lib: Makefile
Added Files:
othersrc/external/bsd/elex/dist: gap.c gap.h
Removed Files:
othersrc/external/bsd/elex/dist: striter.c striter.h
Log Message:
Elex version 20230621
=
+ agcre - added internal magic numbers to agcre to attempt to catch
misbehaving programs overwriting sections of memory (extremely coarse-
grained checks here).
+ agcre - check internal magic numbers before attempting to execute
regex programs
+ agcre - bump agcre magic number in the external structure
+ elex - remove striter (simple) and move to using buffer gap functions
+ elex - fix a bug whereby we check if a rule has a return value
before attempting to parse that return value.
+ elex - fix up tests for all of these fixes
+ elex - error out when reading rules if a bad rule is encountered (as
the resulting lexer would be erroneous if we continued)
+ elex - bump elex version to 20230621
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/TODO
cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/Makefile \
othersrc/external/bsd/elex/dist/agcre.h
cvs rdiff -u -r1.3 -r1.4 othersrc/external/bsd/elex/dist/agcre.c \
othersrc/external/bsd/elex/dist/elex.c \
othersrc/external/bsd/elex/dist/elex.h \
othersrc/external/bsd/elex/dist/main.c
cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/dist/gap.c \
othersrc/external/bsd/elex/dist/gap.h
cvs rdiff -u -r1.3 -r0 othersrc/external/bsd/elex/dist/striter.c
cvs rdiff -u -r1.2 -r0 othersrc/external/bsd/elex/dist/striter.h
cvs rdiff -u -r1.3 -r1.4 othersrc/external/bsd/elex/dist/tests/28.expected
cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/lib/Makefile
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/TODO
diff -u othersrc/external/bsd/elex/TODO:1.1 othersrc/external/bsd/elex/TODO:1.2
--- othersrc/external/bsd/elex/TODO:1.1 Thu Dec 9 04:15:25 2021
+++ othersrc/external/bsd/elex/TODO Wed Jun 21 23:36:17 2023
@@ -44,3 +44,5 @@ get line number action
bookmarks
clone
read-defs from read-grammar
+bug fix - test we have an m[6] match before using it as return value
+move from striter to gap functions
Index: othersrc/external/bsd/elex/dist/Makefile
diff -u othersrc/external/bsd/elex/dist/Makefile:1.2 othersrc/external/bsd/elex/dist/Makefile:1.3
--- othersrc/external/bsd/elex/dist/Makefile:1.2 Tue Apr 25 20:03:39 2023
+++ othersrc/external/bsd/elex/dist/Makefile Wed Jun 21 23:36:17 2023
@@ -1,7 +1,7 @@
PROG= elex
SRCS+= agcre.c
SRCS+= elex.c
-SRCS+= striter.c
+SRCS+= gap.c
SRCS+= main.c
MKMAN= no
WARNS= 5
Index: othersrc/external/bsd/elex/dist/agcre.h
diff -u othersrc/external/bsd/elex/dist/agcre.h:1.2 othersrc/external/bsd/elex/dist/agcre.h:1.3
--- othersrc/external/bsd/elex/dist/agcre.h:1.2 Wed Feb 22 01:01:39 2023
+++ othersrc/external/bsd/elex/dist/agcre.h Wed Jun 21 23:36:17 2023
@@ -53,7 +53,7 @@
#define REG_SUCCESS AGCRE_REG_SUCCESS
#define REG_FAILURE AGCRE_REG_FAILURE
#define REG_NOMATCH AGCRE_REG_FAILURE
-#define REG_MAGIC AGCRE_MAGIC
+#define REG_MAGIC AGCRE_MAGIC2
#define REG_MAX_SUBEXPR AGCRE_MAX_SUBEXPR
#define REG_MAX_EXPR_LENGTH AGCRE_MAX_EXPR_LENGTH
#define REG_ANCHOR AGCRE_REG_ANCHOR
@@ -89,7 +89,7 @@
#define AGCRE_REG_SUCCESS 0
#define AGCRE_REG_FAILURE 1
-#define AGCRE_MAGIC 0x20170801
+#define AGCRE_MAGIC2 0x20230621
/* limits we impose on expressions */
#define AGCRE_MAX_SUBEXPR 100
Index: othersrc/external/bsd/elex/dist/agcre.c
diff -u othersrc/external/bsd/elex/dist/agcre.c:1.3 othersrc/external/bsd/elex/dist/agcre.c:1.4
--- othersrc/external/bsd/elex/dist/agcre.c:1.3 Thu Feb 23 19:36:07 2023
+++ othersrc/external/bsd/elex/dist/agcre.c Wed Jun 21 23:36:17 2023
@@ -159,17 +159,26 @@ typedef struct threadlist_t {
re_thread_t t[1]; /* the threads */
} threadlist_t;
+#define MAGIC1 0xac1deaf0
+#define MAGIC2 0x41525345
+#define MAGIC3 0xd0d0d00d
+#define MAGIC4 0x666f7572
+
/* regular expression internals */
typedef struct re_t {
+ uint32_t magic1; /* magic number #1 */
instr_t *prog; /* start of instructions */
uint32_t instrc; /* # of instructions */
uint32_t gen; /* generation number */
uint32_t setc; /* # of sets */
uint32_t maxset; /* allocated # of sets */
+ uint32_t magic2; /* magic number #2 */
set_t *sets; /* sets */
uint32_t flags; /* comp/exec flags */
context_t *ctxlist; /* list of contexts */
+ uint32_t magic3; /* magic number #3 */
instr_t *pc; /* prog counter */
+ uint32_t magic4; /* magic number #4 */
int msgc; /* # of cha
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Wed Jun 21 23:36:17 UTC 2023 Modified Files: othersrc/external/bsd/elex: TODO othersrc/external/bsd/elex/dist: Makefile agcre.c agcre.h elex.c elex.h main.c othersrc/external/bsd/elex/dist/tests: 28.expected othersrc/external/bsd/elex/lib: Makefile Added Files: othersrc/external/bsd/elex/dist: gap.c gap.h Removed Files: othersrc/external/bsd/elex/dist: striter.c striter.h Log Message: Elex version 20230621 = + agcre - added internal magic numbers to agcre to attempt to catch misbehaving programs overwriting sections of memory (extremely coarse- grained checks here). + agcre - check internal magic numbers before attempting to execute regex programs + agcre - bump agcre magic number in the external structure + elex - remove striter (simple) and move to using buffer gap functions + elex - fix a bug whereby we check if a rule has a return value before attempting to parse that return value. + elex - fix up tests for all of these fixes + elex - error out when reading rules if a bad rule is encountered (as the resulting lexer would be erroneous if we continued) + elex - bump elex version to 20230621 To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/TODO cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/Makefile \ othersrc/external/bsd/elex/dist/agcre.h cvs rdiff -u -r1.3 -r1.4 othersrc/external/bsd/elex/dist/agcre.c \ othersrc/external/bsd/elex/dist/elex.c \ othersrc/external/bsd/elex/dist/elex.h \ othersrc/external/bsd/elex/dist/main.c cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/dist/gap.c \ othersrc/external/bsd/elex/dist/gap.h cvs rdiff -u -r1.3 -r0 othersrc/external/bsd/elex/dist/striter.c cvs rdiff -u -r1.2 -r0 othersrc/external/bsd/elex/dist/striter.h cvs rdiff -u -r1.3 -r1.4 othersrc/external/bsd/elex/dist/tests/28.expected cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/lib/Makefile Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Tue Apr 25 20:03:40 UTC 2023
Modified Files:
othersrc/external/bsd/elex/bin: Makefile
othersrc/external/bsd/elex/dist: Makefile
Added Files:
othersrc/external/bsd/elex/dist/tests: 31.expected 31.in 32.expected
32.in json.lex
Log Message:
Add an example json lexer, and some tests for it.
To generate a diff of this commit:
cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/bin/Makefile
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/Makefile
cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/dist/tests/31.expected \
othersrc/external/bsd/elex/dist/tests/31.in \
othersrc/external/bsd/elex/dist/tests/32.expected \
othersrc/external/bsd/elex/dist/tests/32.in \
othersrc/external/bsd/elex/dist/tests/json.lex
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/bin/Makefile
diff -u othersrc/external/bsd/elex/bin/Makefile:1.2 othersrc/external/bsd/elex/bin/Makefile:1.3
--- othersrc/external/bsd/elex/bin/Makefile:1.2 Fri Feb 24 23:07:54 2023
+++ othersrc/external/bsd/elex/bin/Makefile Tue Apr 25 20:03:39 2023
@@ -143,3 +143,11 @@ t: ${PROG}
env LD_LIBRARY_PATH=${LIB_ELEX_DIR} ./${PROG} -g -f ${DIST}/tests/30.lex ${DIST}/tests/21.in > 30.out
diff ${DIST}/tests/30.expected 30.out
rm -f 30.out
+ @echo "31. json"
+ env LD_LIBRARY_PATH=${LIB_ELEX_DIR} ./${PROG} -f ${DIST}/tests/json.lex ${DIST}/tests/31.in > 31.out
+ diff ${DIST}/tests/31.expected 31.out
+ rm -f 31.out
+ @echo "32. more json"
+ env LD_LIBRARY_PATH=${LIB_ELEX_DIR} ./${PROG} -f ${DIST}/tests/json.lex ${DIST}/tests/32.in > 32.out
+ diff ${DIST}/tests/32.expected 32.out
+ rm -f 32.out
Index: othersrc/external/bsd/elex/dist/Makefile
diff -u othersrc/external/bsd/elex/dist/Makefile:1.1 othersrc/external/bsd/elex/dist/Makefile:1.2
--- othersrc/external/bsd/elex/dist/Makefile:1.1 Thu Dec 9 04:15:26 2021
+++ othersrc/external/bsd/elex/dist/Makefile Tue Apr 25 20:03:39 2023
@@ -1,6 +1,7 @@
PROG= elex
SRCS+= agcre.c
SRCS+= elex.c
+SRCS+= striter.c
SRCS+= main.c
MKMAN= no
WARNS= 5
Added files:
Index: othersrc/external/bsd/elex/dist/tests/31.expected
diff -u /dev/null othersrc/external/bsd/elex/dist/tests/31.expected:1.1
--- /dev/null Tue Apr 25 20:03:40 2023
+++ othersrc/external/bsd/elex/dist/tests/31.expected Tue Apr 25 20:03:40 2023
@@ -0,0 +1,65 @@
+[6] 1 '{'
+[1] 10 '"glossary"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 7 '"title"'
+[9] 1 ':'
+[1] 18 '"example glossary"'
+[8] 1 ','
+[1] 10 '"GlossDiv"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 7 '"title"'
+[9] 1 ':'
+[1] 3 '"S"'
+[8] 1 ','
+[1] 11 '"GlossList"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 12 '"GlossEntry"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 4 '"ID"'
+[9] 1 ':'
+[1] 6 '"SGML"'
+[8] 1 ','
+[1] 8 '"SortAs"'
+[9] 1 ':'
+[1] 6 '"SGML"'
+[8] 1 ','
+[1] 11 '"GlossTerm"'
+[9] 1 ':'
+[1] 38 '"Standard Generalized Markup Language"'
+[8] 1 ','
+[1] 9 '"Acronym"'
+[9] 1 ':'
+[1] 6 '"SGML"'
+[8] 1 ','
+[1] 8 '"Abbrev"'
+[9] 1 ':'
+[1] 15 '"ISO 8879:1986"'
+[8] 1 ','
+[1] 10 '"GlossDef"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 6 '"para"'
+[9] 1 ':'
+[1] 74 '"A meta-markup language, used to create markup languages such as DocBook."'
+[8] 1 ','
+[1] 14 '"GlossSeeAlso"'
+[9] 1 ':'
+[4] 1 '['
+[1] 5 '"GML"'
+[8] 1 ','
+[1] 5 '"XML"'
+[5] 1 ']'
+[7] 1 '}'
+[8] 1 ','
+[1] 10 '"GlossSee"'
+[9] 1 ':'
+[1] 8 '"markup"'
+[7] 1 '}'
+[7] 1 '}'
+[7] 1 '}'
+[7] 1 '}'
+[7] 1 '}'
Index: othersrc/external/bsd/elex/dist/tests/31.in
diff -u /dev/null othersrc/external/bsd/elex/dist/tests/31.in:1.1
--- /dev/null Tue Apr 25 20:03:40 2023
+++ othersrc/external/bsd/elex/dist/tests/31.in Tue Apr 25 20:03:40 2023
@@ -0,0 +1,22 @@
+{
+"glossary": {
+"title": "example glossary",
+ "GlossDiv": {
+"title": "S",
+ "GlossList": {
+"GlossEntry": {
+"ID": "SGML",
+ "SortAs": "SGML",
+ "GlossTerm": "Standard Generalized Markup Language",
+ "Acronym": "SGML",
+ "Abbrev": "ISO 8879:1986",
+ "GlossDef": {
+"para": "A meta-markup language, used to create markup languages such as DocBook.",
+ "GlossSeeAlso": ["GML", "XML"]
+},
+ "GlossSee": "markup"
+}
+}
+}
+}
+}
Index: othersrc/external/bsd/elex/dist/tests/32.expected
diff -u /dev/null othersrc/external/bsd/elex/dist/tests/32.expected:1.1
--- /dev/null Tue Apr 25 20:03:40 2023
+++ othersrc/external/bsd/elex/dist/tests/32.expected Tue Apr 25 20:03:40 2023
@@ -0,0 +1,335 @@
+[6] 1 '{'
+[1] 9 '"web-app"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 9 '"servlet"'
+[9] 1 ':'
+[4] 1 '['
+[6] 1 '{'
+[1] 14 '"servlet-name"'
+[9] 1 ':'
+[1] 10 '"cofaxCDS"'
+[8] 1 ','
+[1] 15 '"servlet-class"'
+[9] 1 ':'
+[1] 26 '"org.cofax.cds.CDSServlet"'
+[8] 1 ','
+[1] 12 '"init-param"'
+[9] 1 ':'
+[6] 1 '{'
+[1] 31 '"configGlossary:installationAt"'
+[9] 1 ':'
+[1] 18 '"Philadelphi
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Tue Apr 25 20:03:40 UTC 2023 Modified Files: othersrc/external/bsd/elex/bin: Makefile othersrc/external/bsd/elex/dist: Makefile Added Files: othersrc/external/bsd/elex/dist/tests: 31.expected 31.in 32.expected 32.in json.lex Log Message: Add an example json lexer, and some tests for it. To generate a diff of this commit: cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/bin/Makefile cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/Makefile cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/dist/tests/31.expected \ othersrc/external/bsd/elex/dist/tests/31.in \ othersrc/external/bsd/elex/dist/tests/32.expected \ othersrc/external/bsd/elex/dist/tests/32.in \ othersrc/external/bsd/elex/dist/tests/json.lex Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Fri Feb 24 23:08:17 UTC 2023 Modified Files: othersrc/external/bsd/elex: README Log Message: document the "namespace" action in the README file To generate a diff of this commit: cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/README Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: othersrc/external/bsd/elex/README diff -u othersrc/external/bsd/elex/README:1.2 othersrc/external/bsd/elex/README:1.3 --- othersrc/external/bsd/elex/README:1.2 Thu Feb 23 19:16:31 2023 +++ othersrc/external/bsd/elex/README Fri Feb 24 23:08:17 2023 @@ -285,6 +285,7 @@ elex_exec_str: allocated space "get-yystate" - get the name of the current state in allocated space "get-yytext" - get the full text of the current match in allocated space +"namespace" - return the name space of the embedded library used (if any) Designing elex definition files ===
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Fri Feb 24 23:08:17 UTC 2023 Modified Files: othersrc/external/bsd/elex: README Log Message: document the "namespace" action in the README file To generate a diff of this commit: cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/README Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Fri Feb 24 23:07:54 UTC 2023
Modified Files:
othersrc/external/bsd/elex/bin: Makefile
othersrc/external/bsd/elex/lib: Makefile
Log Message:
add inclusion of any lib namespace (if there is oneChanges missed from
yesterday's commit
include the library namespace, if there is one
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/bin/Makefile
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/lib/Makefile
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/bin/Makefile
diff -u othersrc/external/bsd/elex/bin/Makefile:1.1 othersrc/external/bsd/elex/bin/Makefile:1.2
--- othersrc/external/bsd/elex/bin/Makefile:1.1 Thu Dec 9 04:15:25 2021
+++ othersrc/external/bsd/elex/bin/Makefile Fri Feb 24 23:07:54 2023
@@ -12,6 +12,8 @@ CPPFLAGS+= -g -O0
LDFLAGS+= -g -O0
.endif
+.sinclude "../namespace.mk"
+
LIB_ELEX_DIR!= cd ${.CURDIR}/../lib && ${PRINTOBJDIR}
DIST= ${.CURDIR}/../dist
Index: othersrc/external/bsd/elex/lib/Makefile
diff -u othersrc/external/bsd/elex/lib/Makefile:1.1 othersrc/external/bsd/elex/lib/Makefile:1.2
--- othersrc/external/bsd/elex/lib/Makefile:1.1 Thu Dec 9 04:15:26 2021
+++ othersrc/external/bsd/elex/lib/Makefile Fri Feb 24 23:07:54 2023
@@ -6,6 +6,8 @@ CPPFLAGS+= -I${DIST}
MKMAN= no
WARNS= 5
+.sinclude "../namespace.mk"
+
# set symbol visibility for .so
CPPFLAGS+= -DHIDE_AGCRE=1
CPPFLAGS+= -DHIDE_STRITER=1
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Fri Feb 24 23:07:54 UTC 2023 Modified Files: othersrc/external/bsd/elex/bin: Makefile othersrc/external/bsd/elex/lib: Makefile Log Message: add inclusion of any lib namespace (if there is oneChanges missed from yesterday's commit include the library namespace, if there is one To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/bin/Makefile cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/lib/Makefile Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Thu Feb 23 19:36:08 UTC 2023
Modified Files:
othersrc/external/bsd/elex/dist: agcre.c elex.c elex.h main.c striter.c
othersrc/external/bsd/elex/dist/tests: 28.expected
Added Files:
othersrc/external/bsd/elex: namespace.mk
Log Message:
Update elex to version 20230223
+ add introspection - add a "namespace" action to elex_exec_str to return any
embedded library namespace used when compiling
+ move LIB_NAMESPACE from embedded definitions to be defined in Makefiles, and
include the makefile snippet if it exists
+ README typo fixes and added clarification from Brad Harder
+ bump version to 20230223
To generate a diff of this commit:
cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/namespace.mk
cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/agcre.c \
othersrc/external/bsd/elex/dist/elex.c \
othersrc/external/bsd/elex/dist/elex.h \
othersrc/external/bsd/elex/dist/main.c \
othersrc/external/bsd/elex/dist/striter.c
cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/tests/28.expected
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/dist/agcre.c
diff -u othersrc/external/bsd/elex/dist/agcre.c:1.2 othersrc/external/bsd/elex/dist/agcre.c:1.3
--- othersrc/external/bsd/elex/dist/agcre.c:1.2 Wed Feb 22 01:01:39 2023
+++ othersrc/external/bsd/elex/dist/agcre.c Thu Feb 23 19:36:07 2023
@@ -36,7 +36,6 @@
#include
#include
-#define LIB_NAMESPACE elex_
#include "agcre.h"
/* callback struct */
Index: othersrc/external/bsd/elex/dist/elex.c
diff -u othersrc/external/bsd/elex/dist/elex.c:1.2 othersrc/external/bsd/elex/dist/elex.c:1.3
--- othersrc/external/bsd/elex/dist/elex.c:1.2 Wed Feb 22 01:01:39 2023
+++ othersrc/external/bsd/elex/dist/elex.c Thu Feb 23 19:36:07 2023
@@ -33,7 +33,6 @@
#include
#include
-#define LIB_NAMESPACE elex_
#include "agcre.h"
#include "striter.h"
#include "elex.h"
@@ -672,13 +671,17 @@ elex_exec(elex_t *elex, const char *info
return 0;
}
+/* used to stringify the namespace we're using */
+#define ELEX_STRINGIFY(_x) ELEX_STRINGIFY2(_x)
+#define ELEX_STRINGIFY2(_x) #_x
+
/* one function to access string values */
void *
elex_exec_str(elex_t *elex, const char *info, uint64_t n, uint64_t *size)
{
uint64_t len;
- if (elex == NULL || info == NULL) {
+ if (elex == NULL || info == NULL || size == NULL) {
return NULL;
}
switch(djbhash(info)) {
@@ -698,6 +701,9 @@ elex_exec_str(elex_t *elex, const char *
return allocate(elex->states[elex->yystate].name, strlen(elex->states[elex->yystate].name), size);
case /* "get-yytext" */ 0x88065864:
return allocate(elex->yytext, elex->yyleng, size);
+ case /* "namespace" */ 0x41041c23:
+ return allocate(ELEX_STRINGIFY(LIB_NAMESPACE),
+ strlen(ELEX_STRINGIFY(LIB_NAMESPACE)), size);
}
return NULL;
}
Index: othersrc/external/bsd/elex/dist/elex.h
diff -u othersrc/external/bsd/elex/dist/elex.h:1.2 othersrc/external/bsd/elex/dist/elex.h:1.3
--- othersrc/external/bsd/elex/dist/elex.h:1.2 Wed Feb 22 01:01:39 2023
+++ othersrc/external/bsd/elex/dist/elex.h Thu Feb 23 19:36:07 2023
@@ -23,7 +23,7 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef ELEX_H_
-#define ELEX_H_ 20230221
+#define ELEX_H_ 20230223
#include
Index: othersrc/external/bsd/elex/dist/main.c
diff -u othersrc/external/bsd/elex/dist/main.c:1.2 othersrc/external/bsd/elex/dist/main.c:1.3
--- othersrc/external/bsd/elex/dist/main.c:1.2 Wed Feb 22 01:01:39 2023
+++ othersrc/external/bsd/elex/dist/main.c Thu Feb 23 19:36:07 2023
@@ -32,7 +32,6 @@
#include
#include
-#define LIB_NAMESPACE elex_
#include "elex.h"
#define STRINGIFY(x) #x
Index: othersrc/external/bsd/elex/dist/striter.c
diff -u othersrc/external/bsd/elex/dist/striter.c:1.2 othersrc/external/bsd/elex/dist/striter.c:1.3
--- othersrc/external/bsd/elex/dist/striter.c:1.2 Wed Feb 22 01:20:52 2023
+++ othersrc/external/bsd/elex/dist/striter.c Thu Feb 23 19:36:07 2023
@@ -33,7 +33,6 @@
#include
#include
-#define LIB_NAMESPACE elex_
#include "striter.h"
/* a string iterator structure */
Index: othersrc/external/bsd/elex/dist/tests/28.expected
diff -u othersrc/external/bsd/elex/dist/tests/28.expected:1.2 othersrc/external/bsd/elex/dist/tests/28.expected:1.3
--- othersrc/external/bsd/elex/dist/tests/28.expected:1.2 Wed Feb 22 01:01:40 2023
+++ othersrc/external/bsd/elex/dist/tests/28.expected Thu Feb 23 19:36:08 2023
@@ -1 +1 @@
-elex version 20230221
+elex version 20230223
Added files:
Index: othersrc/external/bsd/elex/namespace.mk
diff -u /dev/null othersrc/external/bsd/elex/namespace.mk:1.1
--- /dev/null Thu Feb 23 19:36:08 2023
+++ othersrc/external/bsd/elex/namespace.mk Thu Feb 23 19:36:07 2023
@@ -0,0 +1 @@
+CPPFLAGS+= -DLIB_NAMESPACE="elex_"
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Thu Feb 23 19:36:08 UTC 2023 Modified Files: othersrc/external/bsd/elex/dist: agcre.c elex.c elex.h main.c striter.c othersrc/external/bsd/elex/dist/tests: 28.expected Added Files: othersrc/external/bsd/elex: namespace.mk Log Message: Update elex to version 20230223 + add introspection - add a "namespace" action to elex_exec_str to return any embedded library namespace used when compiling + move LIB_NAMESPACE from embedded definitions to be defined in Makefiles, and include the makefile snippet if it exists + README typo fixes and added clarification from Brad Harder + bump version to 20230223 To generate a diff of this commit: cvs rdiff -u -r0 -r1.1 othersrc/external/bsd/elex/namespace.mk cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/agcre.c \ othersrc/external/bsd/elex/dist/elex.c \ othersrc/external/bsd/elex/dist/elex.h \ othersrc/external/bsd/elex/dist/main.c \ othersrc/external/bsd/elex/dist/striter.c cvs rdiff -u -r1.2 -r1.3 othersrc/external/bsd/elex/dist/tests/28.expected Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc Committed By: agc Date: Thu Feb 23 19:16:31 UTC 2023 Modified Files: othersrc/external/bsd/elex: README Log Message: Fix typos and case issues, and properly hyphenate words. From Brad Harder. To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/README Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Thu Feb 23 19:16:31 UTC 2023
Modified Files:
othersrc/external/bsd/elex: README
Log Message:
Fix typos and case issues, and properly hyphenate words. From Brad Harder.
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/README
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/README
diff -u othersrc/external/bsd/elex/README:1.1 othersrc/external/bsd/elex/README:1.2
--- othersrc/external/bsd/elex/README:1.1 Thu Dec 9 04:15:25 2021
+++ othersrc/external/bsd/elex/README Thu Feb 23 19:16:31 2023
@@ -3,7 +3,7 @@ Elex - an embeddable regexp-based lexer
I have found myself fairly often needing a lexer utility to tokenise
input (for configuration files, for various file-scanning utilities,
-and for other applciations), but using full-blown lex(1) program to do
+and for other applications), but using full-blown lex(1) program to do
this is overkill, or designed for a separate process, which doesn't fit
well with the design - syntax-coloring editors, for example.
@@ -35,13 +35,13 @@ another side effect is the ability to us
such as perl escapes, UTF-8 matching, in-subexpression ignore case, etc.
elex implements start states, similar to flex. These are useful for
-recognising multiline comments (almost any language), or multi-line
+recognising multi-line comments (almost any language), or multi-line
strings (perl, python, lua etc).
elex dynamically sizes the regmatch arrays used to accommodate the
largest regexp in the input, and matching subexpressions can be
returned to the caller. The 0'th subexpression is the whole matching
-expression, and is the same as "yytext".
+expression, and is the same as "yytext" in lex(1).
And so on to an elex definition which recognises C and some C++:
@@ -81,7 +81,7 @@ And so on to an elex definition which re
#[ \t]*(define|el(se|if)|endif|error|if|ifn?def|include|line|pragma|undef)[^\n]* { return PREPROC; }
Start states are explicitly used for rules, since it is easier to read
-in practice. Elex comments are eol-style comments, beginning '#' and
+in practice. elex comments are eol-style comments, beginning '#' and
ending with '\n'. Types can be defined using the "%type" directive,
and the unsigned 32bit value they take will be returned. This is more
work than using magic constants, but much more readable in practice -
@@ -102,7 +102,7 @@ Start states can be defined using the %s
transitioned to using the BEGIN() action, in the same way as standard
lex(1).
-Elex provides bookmarks, which are numbered numerically from 0.
+elex provides bookmarks, which are numbered numerically from 0.
Assuming a mark has already been successfully created using
"set-mark", the bookmark offset can be retrieved by using its index
using "get-mark", and the user can then seek to that offset. This
@@ -300,7 +300,7 @@ return to the calling program. Since a
input, it is advised to define return types starting at 1.
Historically, in lex definitions, the user-defined types started at
256, and it was common to return ASCII values for single characters up
-to 256. Since this is no longer acceptable in a world with multibyte
+to 256. Since this is no longer acceptable in a world with multi-byte
characters, and because we tend to tokenise based on types of input
tokens, hopefully this practice will never be used again.
@@ -330,7 +330,7 @@ in this.
Usually, when tokenising programming language, there would be a number
of definitions for reserved words, and standard identifiers. There would
also be definitions for punctuation, and numeric and string constants.
-Some languages have definitions for multiline strings.
+Some languages have definitions for multi-line strings.
Alistair Croooks
Thu Nov 18 16:57:44 PST 2021
CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc Committed By: agc Date: Wed Feb 22 01:20:52 UTC 2023 Modified Files: othersrc/external/bsd/elex/dist: striter.c striter.h Log Message: Commit elex string iterator changes missed in previous commit: + move to a more extensible embedded library namespace protection To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/striter.c \ othersrc/external/bsd/elex/dist/striter.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc
Committed By: agc
Date: Wed Feb 22 01:20:52 UTC 2023
Modified Files:
othersrc/external/bsd/elex/dist: striter.c striter.h
Log Message:
Commit elex string iterator changes missed in previous commit:
+ move to a more extensible embedded library namespace protection
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/striter.c \
othersrc/external/bsd/elex/dist/striter.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/dist/striter.c
diff -u othersrc/external/bsd/elex/dist/striter.c:1.1 othersrc/external/bsd/elex/dist/striter.c:1.2
--- othersrc/external/bsd/elex/dist/striter.c:1.1 Thu Dec 9 04:15:26 2021
+++ othersrc/external/bsd/elex/dist/striter.c Wed Feb 22 01:20:52 2023
@@ -33,6 +33,7 @@
#include
#include
+#define LIB_NAMESPACE elex_
#include "striter.h"
/* a string iterator structure */
@@ -102,14 +103,14 @@ addtext(striter_t *str, const char *s, u
/*/
/* make a new string */
-STRITER_EXPORT striter_t *
+striter_t *
striter_new(void)
{
return calloc(1, sizeof(striter_t));
}
/* dispose of string */
-STRITER_EXPORT int
+int
striter_dispose(striter_t **str)
{
if (str && *str) {
@@ -122,7 +123,7 @@ striter_dispose(striter_t **str)
}
/* command with integer return */
-STRITER_EXPORT int
+int
striter_exec(striter_t *str, const char *info, const char *s, uint64_t n)
{
if (str == NULL || info == NULL) {
@@ -136,7 +137,7 @@ striter_exec(striter_t *str, const char
}
/* command with char string return */
-STRITER_EXPORT char *
+char *
striter_exec_mem(striter_t *str, const char *info, size_t *size)
{
if (str == NULL || info == NULL || size == NULL) {
Index: othersrc/external/bsd/elex/dist/striter.h
diff -u othersrc/external/bsd/elex/dist/striter.h:1.1 othersrc/external/bsd/elex/dist/striter.h:1.2
--- othersrc/external/bsd/elex/dist/striter.h:1.1 Thu Dec 9 04:15:26 2021
+++ othersrc/external/bsd/elex/dist/striter.h Wed Feb 22 01:20:52 2023
@@ -23,42 +23,22 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef STRITER_H_
-#define STRITER_H_ 20211121
+#define STRITER_H_ 20230222
#include
-struct striter_t;
-typedef struct striter_t striter_t;
-
-#ifndef USE_VISIBILITY
-# if defined(__GNUC__)
-#if __GNUC__ >= 4
-#define USE_VISIBILITY 1
-#else
-#define USE_VISIBILITY 0
-#endif
-# else
-#define USE_VISIBILITY 0
-# endif
-#endif
-
-#if USE_VISIBILITY
-# define DLL_PUBLIC __attribute__ ((visibility ("default")))
-# define DLL_LOCAL __attribute__ ((visibility ("hidden")))
-#else
-# define DLL_PUBLIC
-# define DLL_LOCAL
-#endif
-
-#ifndef HIDE_STRITER
-#define HIDE_STRITER 0
+#ifdef LIB_NAMESPACE
+#define SI_CONCAT(x, y) x##y
+#define SI_NAMESPACE(x, y) SI_CONCAT(x, y)
+#define striter_t SI_NAMESPACE(LIB_NAMESPACE, striter_t)
+#define striter_new SI_NAMESPACE(LIB_NAMESPACE, striter_new)
+#define striter_dispose SI_NAMESPACE(LIB_NAMESPACE, striter_dispose)
+#define striter_exec SI_NAMESPACE(LIB_NAMESPACE, striter_exec)
+#define striter_exec_mem SI_NAMESPACE(LIB_NAMESPACE, striter_exec_mem)
#endif
-#if HIDE_STRITER
-#define STRITER_EXPORT DLL_LOCAL
-#else
-#define STRITER_EXPORT DLL_PUBLIC
-#endif
+struct striter_t;
+typedef struct striter_t striter_t;
#ifndef __BEGIN_DECLS
# if defined(__cplusplus)
@@ -72,10 +52,10 @@ typedef struct striter_t striter_t;
__BEGIN_DECLS
-STRITER_EXPORT striter_t *striter_new(void);
-STRITER_EXPORT int striter_dispose(striter_t **/*str*/);
-STRITER_EXPORT int striter_exec(striter_t */*str*/, const char */*info*/, const char */*s*/, uint64_t /*n*/);
-STRITER_EXPORT char *striter_exec_mem(striter_t */*str*/, const char */*info*/, size_t */*size*/);
+striter_t *striter_new(void);
+int striter_dispose(striter_t **/*str*/);
+int striter_exec(striter_t */*str*/, const char */*info*/, const char */*s*/, uint64_t /*n*/);
+char *striter_exec_mem(striter_t */*str*/, const char */*info*/, size_t */*size*/);
__END_DECLS
CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc
Committed By: agc
Date: Wed Feb 22 01:01:40 UTC 2023
Modified Files:
othersrc/external/bsd/elex/dist: agcre.c agcre.h elex.c elex.h main.c
othersrc/external/bsd/elex/dist/tests: 28.expected
Log Message:
Update to elex version 20230221
20230221
+ protect elex embedded name space in a more extensible way
+ properly protect elex_make_new_rule
+ don't filter on symbol visibility, use LIB_NAMESPACE definition
+ sync agcre with bug fixes and reverse searching and other functionality
updates
+ when searching elex start state names, use a hash value
+ bump version to 20230221
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/agcre.c \
othersrc/external/bsd/elex/dist/agcre.h \
othersrc/external/bsd/elex/dist/elex.c \
othersrc/external/bsd/elex/dist/elex.h \
othersrc/external/bsd/elex/dist/main.c
cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/tests/28.expected
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: othersrc/external/bsd/elex/dist/agcre.c
diff -u othersrc/external/bsd/elex/dist/agcre.c:1.1 othersrc/external/bsd/elex/dist/agcre.c:1.2
--- othersrc/external/bsd/elex/dist/agcre.c:1.1 Thu Dec 9 04:15:26 2021
+++ othersrc/external/bsd/elex/dist/agcre.c Wed Feb 22 01:01:39 2023
@@ -36,6 +36,7 @@
#include
#include
+#define LIB_NAMESPACE elex_
#include "agcre.h"
/* callback struct */
@@ -223,6 +224,7 @@ static int unicode_isOther_Uppercase(uin
static int unicode_isPattern_White_Space(uint32_t /*ch*/);
static int unicode_isalnum(uint32_t /*ch*/);
static int unicode_isalpha(uint32_t /*ch*/);
+static int unicode_ispunct2(uint32_t /*ch*/);
static int unicode_isblank(uint32_t /*ch*/);
static int unicode_iscntrl(uint32_t /*ch*/);
static int unicode_isdigit(uint32_t /*ch*/);
@@ -233,7 +235,7 @@ static int unicode_ispunct(uint32_t /*ch
static int unicode_isspace(uint32_t /*ch*/);
static int unicode_isupper(uint32_t /*ch*/);
static int unicode_isxdigit(uint32_t /*ch*/);
-static int unicode_isident(uint32_t /*ch*/);
+static int unicode_isword(uint32_t /*ch*/);
static uint32_t unicode_tolower(uint32_t /*ch*/);
static uint32_t unicode_toupper(uint32_t /*ch*/);
@@ -417,7 +419,7 @@ emit(re_t *re, retoken_t *tok)
(tok->ch == 'd' || tok->ch == 'D') ? unicode_isdigit :
(tok->ch == 'p' || tok->ch == 'P') ? unicode_isprint :
(tok->ch == 's' || tok->ch == 'S') ? unicode_isspace :
-unicode_isident,
+unicode_isword,
unicode_isupper(tok->ch));
re->pc++;
return 1;
@@ -1169,16 +1171,16 @@ isendline(re_t *re, input_t *in)
static inline int
isbegword(re_t *re, input_t *in)
{
- return (isbegline(re, in) || !unicode_isident(in->prevch)) &&
- unicode_isident(in->ch);
+ return (isbegline(re, in) || !unicode_isword(in->prevch)) &&
+ unicode_isword(in->ch);
}
/* return 1 at end of words */
static inline int
isendword(re_t *re, input_t *in)
{
- return (isendline(re, in) || (in->c > in->so && unicode_isident(in->prevch))) &&
- (!unicode_isident(in->ch));
+ return (isendline(re, in) || (in->c > in->so && unicode_isword(in->prevch))) &&
+ (!unicode_isword(in->ch));
}
/* do the chars match? */
@@ -1577,10 +1579,6 @@ rec_posix_class(input_t *in, set_t *set)
set_add_callback(set, unicode_isgraph, 0);
in->c += 8;
return 1;
- case /* ":ident:]" */ 0x8a1572f1:
- set_add_callback(set, unicode_isident, 0);
- in->c += 8;
- return 1;
case /* ":lower:]" */ 0x8bfc6af8:
set_add_callback(set, unicode_islower, 0);
in->c += 8;
@@ -1593,6 +1591,10 @@ rec_posix_class(input_t *in, set_t *set)
set_add_callback(set, unicode_ispunct, 0);
in->c += 8;
return 1;
+ case /* ":punct2:]" */ 0xf09af6aa:
+ set_add_callback(set, unicode_ispunct2, 0);
+ in->c += 9;
+ return 1;
case /* ":space:]" */ 0xa876bcf2:
set_add_callback(set, unicode_isspace, 0);
in->c += 8;
@@ -1605,6 +1607,10 @@ rec_posix_class(input_t *in, set_t *set)
set_add_callback(set, unicode_isxdigit, 0);
in->c += 9;
return 1;
+ case /* ":word:]" */ 0x96b11914:
+ set_add_callback(set, unicode_isword, 0);
+ in->c += 7;
+ return 1;
default:
return 0;
}
@@ -1661,7 +1667,7 @@ rec_set(input_t *in)
(token->ch == 'd' || token->ch == 'D') ? unicode_isdigit :
(token->ch == 'p' || token->ch == 'P') ? unicode_isprint :
(token->ch == 's' || token->ch == 'S') ? unicode_isspace :
- unicode_isident,
+ unicode_isword,
unicode_isupper((uint8_t)token->ch));
break;
}
@@ -2612,11 +2618,20 @@ unicode_isxdigit(uint32_t ch)
}
static int
-unicode_isident(uint32_t ch)
+unicode_isword(uint32_t ch)
{
return unicode_isalnum(ch) || ch == '_';
}
+static int
+unicode_ispunct2(uint32_t ch)
+{
+ if (!unicode_isprint(ch) || unicode_isspace(ch) || unicode_isword(ch)) {
+ return 0;
+ }
+ return 1;
+}
+
static uint32_t
unicode_tolower(ui
CVS commit: othersrc/external/bsd/elex/dist
Module Name:othersrc Committed By: agc Date: Wed Feb 22 01:01:40 UTC 2023 Modified Files: othersrc/external/bsd/elex/dist: agcre.c agcre.h elex.c elex.h main.c othersrc/external/bsd/elex/dist/tests: 28.expected Log Message: Update to elex version 20230221 20230221 + protect elex embedded name space in a more extensible way + properly protect elex_make_new_rule + don't filter on symbol visibility, use LIB_NAMESPACE definition + sync agcre with bug fixes and reverse searching and other functionality updates + when searching elex start state names, use a hash value + bump version to 20230221 To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/agcre.c \ othersrc/external/bsd/elex/dist/agcre.h \ othersrc/external/bsd/elex/dist/elex.c \ othersrc/external/bsd/elex/dist/elex.h \ othersrc/external/bsd/elex/dist/main.c cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/tests/28.expected Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex/dist/tests
Module Name:othersrc Committed By: agc Date: Thu Dec 9 04:18:00 UTC 2021 Modified Files: othersrc/external/bsd/elex/dist/tests: 14.expected 14.in 20.expected 20.in Log Message: Omit RCS Ids from test input and expected output files. To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/tests/14.expected \ othersrc/external/bsd/elex/dist/tests/14.in \ othersrc/external/bsd/elex/dist/tests/20.expected \ othersrc/external/bsd/elex/dist/tests/20.in Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: othersrc/external/bsd/elex/dist/tests/14.expected diff -u othersrc/external/bsd/elex/dist/tests/14.expected:1.1 othersrc/external/bsd/elex/dist/tests/14.expected:1.2 --- othersrc/external/bsd/elex/dist/tests/14.expected:1.1 Thu Dec 9 04:15:26 2021 +++ othersrc/external/bsd/elex/dist/tests/14.expected Thu Dec 9 04:18:00 2021 @@ -2,10 +2,6 @@ [1] 2 ' ' -[3] 61 '# $NetBSD: 14.expected,v 1.1 2021/12/09 04:15:26 agc Exp $' -[1] 2 ' - -' [3] 53 '# Copyright (c) 2013 Alistair Crooks ' [1] 1 ' ' Index: othersrc/external/bsd/elex/dist/tests/14.in diff -u othersrc/external/bsd/elex/dist/tests/14.in:1.1 othersrc/external/bsd/elex/dist/tests/14.in:1.2 --- othersrc/external/bsd/elex/dist/tests/14.in:1.1 Thu Dec 9 04:15:26 2021 +++ othersrc/external/bsd/elex/dist/tests/14.in Thu Dec 9 04:18:00 2021 @@ -1,7 +1,5 @@ #! /bin/sh -# $NetBSD: 14.in,v 1.1 2021/12/09 04:15:26 agc Exp $ - # Copyright (c) 2013 Alistair Crooks # All rights reserved. # Index: othersrc/external/bsd/elex/dist/tests/20.expected diff -u othersrc/external/bsd/elex/dist/tests/20.expected:1.1 othersrc/external/bsd/elex/dist/tests/20.expected:1.2 --- othersrc/external/bsd/elex/dist/tests/20.expected:1.1 Thu Dec 9 04:15:26 2021 +++ othersrc/external/bsd/elex/dist/tests/20.expected Thu Dec 9 04:18:00 2021 @@ -1,9 +1,6 @@ [3] 9 '#!/bin/sh' [1] 1 ' ' -[3] 91 '# $Header: /cvsroot/othersrc/external/bsd/elex/dist/tests/20.expected,v 1.1 2021/12/09 04:15:26 agc Exp $' -[1] 1 ' -' [3] 1 '#' [1] 1 ' ' Index: othersrc/external/bsd/elex/dist/tests/20.in diff -u othersrc/external/bsd/elex/dist/tests/20.in:1.1 othersrc/external/bsd/elex/dist/tests/20.in:1.2 --- othersrc/external/bsd/elex/dist/tests/20.in:1.1 Thu Dec 9 04:15:26 2021 +++ othersrc/external/bsd/elex/dist/tests/20.in Thu Dec 9 04:18:00 2021 @@ -1,5 +1,4 @@ #!/bin/sh -# $Header: /cvsroot/othersrc/external/bsd/elex/dist/tests/20.in,v 1.1 2021/12/09 04:15:26 agc Exp $ # # Install Perl scripts, adjusting for the correct pathname # $1 = name of perl program
CVS commit: othersrc/external/bsd/elex/dist/tests
Module Name:othersrc Committed By: agc Date: Thu Dec 9 04:18:00 UTC 2021 Modified Files: othersrc/external/bsd/elex/dist/tests: 14.expected 14.in 20.expected 20.in Log Message: Omit RCS Ids from test input and expected output files. To generate a diff of this commit: cvs rdiff -u -r1.1 -r1.2 othersrc/external/bsd/elex/dist/tests/14.expected \ othersrc/external/bsd/elex/dist/tests/14.in \ othersrc/external/bsd/elex/dist/tests/20.expected \ othersrc/external/bsd/elex/dist/tests/20.in Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
CVS commit: othersrc/external/bsd/elex
Module Name:othersrc
Committed By: agc
Date: Thu Dec 9 04:15:26 UTC 2021
Added Files:
othersrc/external/bsd/elex: Makefile README TODO
othersrc/external/bsd/elex/bin: Makefile
othersrc/external/bsd/elex/dist: Makefile agcre.c agcre.h elex.c elex.h
main.c striter.c striter.h
othersrc/external/bsd/elex/dist/tests: 1.expected 1.in 1.lex
10.expected 10.in 11.expected 11.in 12.expected 12.lex 13.expected
13.lex 14.expected 14.in 14.lex 15.expected 15.in 15.lex
16.expected 16.in 16.lex 17.expected 17.in 17.lex 18.expected 18.in
18.lex 19.expected 19.in 19.lex 2.expected 2.in 20.expected 20.in
20.lex 21.expected 21.in 21.lex 22.expected 22.lex 23.expected
23.lex 24.expected 24.lex 25.expected 25.lex 26.expected 26.lex
27.expected 27.lex 28.expected 29.expected 29.in 29.lex 3.expected
3.in 3.lex 30.expected 30.lex 4.expected 4.lex 5.expected 5.in
5.lex 6.expected 6.in 6.lex 7.expected 7.in 7.lex 8.expected 8.in
8.lex 9.expected 9.in 9.lex
othersrc/external/bsd/elex/lib: Makefile shlib_version
Log Message:
Elex - an embeddable regexp-based lexer
===
I have found myself fairly often needing a lexer utility to tokenise
input (for configuration files, for various file-scanning utilities,
and for other applciations), but using full-blown lex(1) program to do
this is overkill, or designed for a separate process, which doesn't fit
well with the design - syntax-coloring editors, for example.
This utility, elex, is a regexp-based tokenizer, an embedded lexer,
which can be used for various uses. It usually takes a lexer file
(similar to lex input files), although the API allows lexers to be
built on the fly by just issuing the calls to make new rules.
Implementation
==
Normal lex(1) is implemented (usually) as a conglomeration of all the
regular expressions for a start state - if multiple matches are found,
the largest match is the one used. I've found that, in practice, this
constrains the way a number of things are done. So I've implemented
elex, using a multiple, prioritised multiple regexp matching scheme.
This supercedes the usual way of distinguishing reserved words and
identifiers in the lexer - recognising all "words" first, and
searching for each word through a number of tables; if a match is not
found, then the word recognized is an identifier. elex works around
this by using a regexp to match reserved words first, and then to
recognise the word as an identifier after that. Since normal regular
expressions usually progress through the input trying to find a match,
the regular expressions used in elex are constrained by anchoring the
search, not allowing progression through the input. In practice, this
makes for more efficient matching.
another side effect is the ability to use more modern regexp features,
such as perl escapes, UTF-8 matching, in-subexpression ignore case, etc.
elex implements start states, similar to flex. These are useful for
recognising multiline comments (almost any language), or multi-line
strings (perl, python, lua etc).
elex dynamically sizes the regmatch arrays used to accommodate the
largest regexp in the input, and matching subexpressions can be
returned to the caller. The 0'th subexpression is the whole matching
expression, and is the same as "yytext".
And so on to an elex definition which recognises C and some C++:
# start state
%state COMMENT
# the types we define
%type IDENT 0xdb8ea4d
%type PUNCT 0xe454e3a
%type NUMBER0xca1edaec
%type COMMENT 0xee5ae423
%type CONSTANT 0xd497741f
%type PREPROC 0xdcf9b98d
%type RESWORD1 0xb5ac6a6a
%type RESWORD2 0xb5ac6a6b
# and finally... the rules
(auto|char|class|const|double|enum|extern|float|friend|inline|int|long|mutable|namespace|new|private|protected|public|register|requires|short|signed|static|this|struct|this|typedef|union|unsigned|void|volatile)\>
{ return RESWORD1; }
(asm|break|case|catch|continue|default|do|else|for|goto|if|return|switch|throw|try|while)\>
{ return RESWORD2; }
[a-zA-Z_][0-9a-zA-Z_]* { return IDENT; }
([1-9][0-9]*|0x[0-9a-f]|0X[0-9A-F]+|0[0-7]*|'(\\.|[^'])*')
{ return NUMBER; }
[ \t\n\r]+ { return PUNCT; }
/\* { BEGIN(COMMENT); return
COMMENT; }
[^\n]*\*/ { BEGIN(INITIAL); return
COMMENT; }
\n|[^\n]+ { return COMMENT; }
//[^\n]*{ return COMMENT; }
"(\\.|[^"])*" { return CONSTANT; }
(==|[-]>|!=|<=|>=|~=|%=|&=|[*]=|[-]=|[+]=|[|]=|(<<|>>)=?)
{ return PUNCT; }
[\u005b;(){}\u005d*<>,+/%~!\u005e&=|.?:\u002d]
{ return P
