[CALCITE-1241] Add a freemarker variable for adding non reserved keyword list to Parser.jj template (Venki Korukanti)
Add a test to prevent unintentionally adding reserved keywords. Close apache/calcite#233 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/20ba4346 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/20ba4346 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/20ba4346 Branch: refs/heads/master Commit: 20ba43461564c983557e9d9fd1b3e04d75ef7b18 Parents: 6c633cf Author: vkorukanti <[email protected]> Authored: Wed Mar 9 11:00:26 2016 -0800 Committer: Julian Hyde <[email protected]> Committed: Sun May 22 12:46:46 2016 -0700 ---------------------------------------------------------------------- core/src/main/codegen/config.fmpp | 7 +- core/src/main/codegen/templates/Parser.jj | 3 + .../calcite/sql/parser/SqlParserTest.java | 343 +++++++++++++++++++ 3 files changed, 352 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/main/codegen/config.fmpp ---------------------------------------------------------------------- diff --git a/core/src/main/codegen/config.fmpp b/core/src/main/codegen/config.fmpp index 6f88718..87ed18f 100644 --- a/core/src/main/codegen/config.fmpp +++ b/core/src/main/codegen/config.fmpp @@ -43,10 +43,15 @@ data: { imports: [ ] - # List of new keywords. Example: "DATABASES", "TABLES". + # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is not a reserved + # keyword add it to 'nonReservedKeywords' section. keywords: [ ] + # List of keywords from "keywords" section that are not reserved. + nonReservedKeywords: [ + ] + # List of methods for parsing custom SQL statements. # Return type of method implementation should be 'SqlNode'. # Example: SqlShowDatabases(), SqlShowTables(). http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/main/codegen/templates/Parser.jj ---------------------------------------------------------------------- diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index bad7622..cfd9f22 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -5619,6 +5619,9 @@ String CommonNonReservedKeyWord() : | <WRITE> | <XML> | <ZONE> + <#list parser.nonReservedKeywords as keyword> + | <${keyword}> + </#list> ) { return unquotedIdentifier(); http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java index 9fe216a..f8bb2b3 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -42,12 +42,14 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -60,6 +62,319 @@ import static org.junit.Assert.assertTrue; public class SqlParserTest { //~ Static fields/initializers --------------------------------------------- + /** + * List of reserved keywords in parser. If a new <b>reserved</b> keyword is added to the + * parser, it must be included in this list. If the keyword is not intended to be a + * reserved keyword, add it to the non-reserved keyword list in the parser. + */ + private static final List<String> RESERVED_KEYWORDS = ImmutableList.of( + "ABS", + "ALL", + "ALLOCATE", + "ALLOW", + "ALTER", + "AND", + "ANY", + "ARE", + "ARRAY", + "AS", + "ASENSITIVE", + "ASYMMETRIC", + "AT", + "ATOMIC", + "AUTHORIZATION", + "AVG", + "BEGIN", + "BETWEEN", + "BIGINT", + "BINARY", + "BIT", + "BLOB", + "BOOLEAN", + "BOTH", + "BY", + "CALL", + "CALLED", + "CARDINALITY", + "CASCADED", + "CASE", + "CAST", + "CEIL", + "CEILING", + "CHAR", + "CHARACTER", + "CHARACTER_LENGTH", + "CHAR_LENGTH", + "CHECK", + "CLOB", + "CLOSE", + "COALESCE", + "COLLATE", + "COLLECT", + "COLUMN", + "COMMIT", + "CONDITION", + "CONNECT", + "CONSTRAINT", + "CONVERT", + "CORR", + "CORRESPONDING", + "COUNT", + "COVAR_POP", + "COVAR_SAMP", + "CREATE", + "CROSS", + "CUBE", + "CUME_DIST", + "CURRENT", + "CURRENT_CATALOG", + "CURRENT_DATE", + "CURRENT_DEFAULT_TRANSFORM_GROUP", + "CURRENT_PATH", + "CURRENT_ROLE", + "CURRENT_SCHEMA", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "CURRENT_USER", + "CURSOR", + "CYCLE", + "DATE", + "DAY", + "DEALLOCATE", + "DEC", + "DECIMAL", + "DECLARE", + "DEFAULT", + "DELETE", + "DENSE_RANK", + "DEREF", + "DESCRIBE", + "DETERMINISTIC", + "DISALLOW", + "DISCONNECT", + "DISTINCT", + "DOUBLE", + "DROP", + "DYNAMIC", + "EACH", + "ELEMENT", + "ELSE", + "END", + "END-EXEC", + "ESCAPE", + "EVERY", + "EXCEPT", + "EXEC", + "EXECUTE", + "EXISTS", + "EXP", + "EXPLAIN", + "EXTEND", + "EXTERNAL", + "EXTRACT", + "FALSE", + "FETCH", + "FILTER", + "FIRST_VALUE", + "FLOAT", + "FLOOR", + "FOR", + "FOREIGN", + "FREE", + "FROM", + "FULL", + "FUNCTION", + "FUSION", + "GET", + "GLOBAL", + "GRANT", + "GROUP", + "GROUPING", + "HAVING", + "HOLD", + "HOUR", + "IDENTITY", + "IMPORT", + "IN", + "INDICATOR", + "INNER", + "INOUT", + "INSENSITIVE", + "INSERT", + "INT", + "INTEGER", + "INTERSECT", + "INTERSECTION", + "INTERVAL", + "INTO", + "IS", + "JOIN", + "LANGUAGE", + "LARGE", + "LAST_VALUE", + "LATERAL", + "LEADING", + "LEFT", + "LIKE", + "LIMIT", + "LN", + "LOCAL", + "LOCALTIME", + "LOCALTIMESTAMP", + "LOWER", + "MATCH", + "MAX", + "MEMBER", + "MERGE", + "METHOD", + "MIN", + "MINUTE", + "MOD", + "MODIFIES", + "MODULE", + "MONTH", + "MULTISET", + "NATIONAL", + "NATURAL", + "NCHAR", + "NCLOB", + "NEW", + "NEXT", + "NO", + "NONE", + "NORMALIZE", + "NOT", + "NULL", + "NULLIF", + "NUMERIC", + "OCTET_LENGTH", + "OF", + "OFFSET", + "OLD", + "ON", + "ONLY", + "OPEN", + "OR", + "ORDER", + "OUT", + "OUTER", + "OVER", + "OVERLAPS", + "OVERLAY", + "PARAMETER", + "PARTITION", + "PERCENTILE_CONT", + "PERCENTILE_DISC", + "PERCENT_RANK", + "POSITION", + "POWER", + "PRECISION", + "PREPARE", + "PRIMARY", + "PROCEDURE", + "RANGE", + "RANK", + "READS", + "REAL", + "RECURSIVE", + "REF", + "REFERENCES", + "REFERENCING", + "REGR_AVGX", + "REGR_AVGY", + "REGR_COUNT", + "REGR_INTERCEPT", + "REGR_R2", + "REGR_SLOPE", + "REGR_SXX", + "REGR_SXY", + "REGR_SYY", + "RELEASE", + "RESET", + "RESULT", + "RETURN", + "RETURNS", + "REVOKE", + "RIGHT", + "ROLLBACK", + "ROLLUP", + "ROW", + "ROWS", + "ROW_NUMBER", + "SAVEPOINT", + "SCOPE", + "SCROLL", + "SEARCH", + "SECOND", + "SELECT", + "SENSITIVE", + "SESSION_USER", + "SET", + "SIMILAR", + "SMALLINT", + "SOME", + "SPECIFIC", + "SPECIFICTYPE", + "SQL", + "SQLEXCEPTION", + "SQLSTATE", + "SQLWARNING", + "SQRT", + "START", + "STATIC", + "STDDEV_POP", + "STDDEV_SAMP", + "STREAM", + "SUBMULTISET", + "SUBSTRING", + "SUM", + "SYMMETRIC", + "SYSTEM", + "SYSTEM_USER", + "TABLE", + "TABLESAMPLE", + "THEN", + "TIME", + "TIMESTAMP", + "TIMEZONE_HOUR", + "TIMEZONE_MINUTE", + "TINYINT", + "TO", + "TRAILING", + "TRANSLATE", + "TRANSLATION", + "TREAT", + "TRIGGER", + "TRIM", + "TRUE", + "UESCAPE", + "UNION", + "UNIQUE", + "UNKNOWN", + "UNNEST", + "UPDATE", + "UPPER", + "UPSERT", + "USER", + "USING", + "VALUE", + "VALUES", + "VARBINARY", + "VARCHAR", + "VARYING", + "VAR_POP", + "VAR_SAMP", + "WHEN", + "WHENEVER", + "WHERE", + "WIDTH_BUCKET", + "WINDOW", + "WITH", + "WITHIN", + "WITHOUT", + "YEAR"); + private static final String ANY = "(?s).*"; private static final ThreadLocal<boolean[]> LINUXIFY = @@ -143,6 +458,10 @@ public class SqlParserTest { getTester().checkExpFails(sql, expectedMsgPattern); } + protected List<String> getReservedKeywords() { + return RESERVED_KEYWORDS; + } + /** * Tests that when there is an error, non-reserved keywords such as "A", * "ABSOLUTE" (which naturally arise whenever a production uses @@ -5851,6 +6170,30 @@ public class SqlParserTest { assertTrue(!jdbcKeywords.contains(",SELECT,")); } + /** + * Tests that reserved keywords are not added to the parser unintentionally. + * (Most keywords are non-reserved. The set of reserved words generally + * only changes with a new version of the SQL standard.) + * + * <p>If the new keyword added is intended to be a reserved keyword, update + * the {@link #RESERVED_KEYWORDS} list. If not, add the keyword to the + * non-reserved keyword list in the parser. + */ + @Test public void testNoUnintendedNewReservedKeywords() { + final SqlAbstractParserImpl.Metadata metadata = getParserMetadata(); + + final List<String> reservedKeywords = new ArrayList<>(); + for (String s : metadata.getTokens()) { + if (metadata.isKeyword(s) && metadata.isReservedWord(s)) { + reservedKeywords.add(s); + } + } + + assertThat("At least one new reserved keyword is added in parser. " + + "Make sure to check the new keywords are intended to be a reserved keywords.", + reservedKeywords, is(getReservedKeywords())); + } + /** Generates a copy of {@code reference.md} with the current set of key * words. Fails if the copy is different from the original. */ @Test public void testGenerateKeyWords() throws IOException {
