This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d6cb698a799 HIVE-28446: Convert some reserved words to non-reserved
words (Shohei Okumiya, reviewed by Denys Kuzmenko)
d6cb698a799 is described below
commit d6cb698a799c8f3d8c3acb61dabb4b958aa31f9d
Author: okumin <[email protected]>
AuthorDate: Mon Oct 28 20:17:16 2024 +0900
HIVE-28446: Convert some reserved words to non-reserved words (Shohei
Okumiya, reviewed by Denys Kuzmenko)
Closes #5394
---
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 5 -
.../hadoop/hive/ql/parse/IdentifiersParser.g | 21 ++-
.../hadoop/hive/ql/parse/TestNonReservedWords.java | 63 +++++++
.../hadoop/hive/ql/parse/TestReservedWords.java | 205 +++++++++++++++++++++
4 files changed, 283 insertions(+), 11 deletions(-)
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 745c437d41b..cf3067a6f22 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -263,11 +263,8 @@ TOK_TABLEROWFORMATLINES;
TOK_TABLEROWFORMATNULL;
TOK_TABLEFILEFORMAT;
TOK_FILEFORMAT_GENERIC;
-TOK_OFFLINE;
TOK_ENABLE;
TOK_DISABLE;
-TOK_READONLY;
-TOK_NO_DROP;
TOK_STORAGEHANDLER;
TOK_NOT_CLUSTERED;
TOK_NOT_SORTED;
@@ -2438,9 +2435,7 @@ primitiveType
| KW_DATETIME -> TOK_DATETIME
| KW_TIMESTAMP -> TOK_TIMESTAMP
| KW_TIMESTAMPLOCALTZ -> TOK_TIMESTAMPLOCALTZ
- //| KW_TIMESTAMPTZ -> TOK_TIMESTAMPTZ
| KW_TIMESTAMP KW_WITH KW_LOCAL KW_TIME KW_ZONE -> TOK_TIMESTAMPLOCALTZ
- //| KW_TIMESTAMP KW_WITH KW_TIME KW_ZONE -> TOK_TIMESTAMPTZ
// Uncomment to allow intervals as table column types
//| KW_INTERVAL KW_YEAR KW_TO KW_MONTH -> TOK_INTERVAL_YEAR_MONTH
//| KW_INTERVAL KW_DAY KW_TO KW_SECOND -> TOK_INTERVAL_DAY_TIME
diff --git
a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index 37b7c4b0ee0..deeb707ef44 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -936,12 +936,12 @@ nonReserved
| KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DCPROPERTIES |
KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY
| KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE |
KW_DISTRIBUTED | KW_DOW | KW_ELEM_TYPE
| KW_ENABLE | KW_ENFORCED | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN |
KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT
- | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOLD_DDLTIME |
KW_HOUR | KW_IDXPROPERTIES | KW_RESPECT | KW_IGNORE
+ | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOUR |
KW_IDXPROPERTIES | KW_RESPECT | KW_IGNORE
| KW_INDEX | KW_INDEXES | KW_INPATH | KW_INPUTDRIVER | KW_INPUTFORMAT |
KW_ITEMS | KW_JAR | KW_JOINCOST | KW_KILL
| KW_KEYS | KW_KEY_TYPE | KW_LAST | KW_LIMIT | KW_OFFSET | KW_LINES |
KW_LOAD | KW_LOCATION | KW_LOCK | KW_LOCKS | KW_LOGICAL | KW_LONG | KW_MANAGED
- | KW_MANAGEDLOCATION | KW_MAPJOIN | KW_MATERIALIZED | KW_METADATA |
KW_MINUTE | KW_MONTH | KW_MSCK | KW_NOSCAN | KW_NO_DROP | KW_NULLS | KW_OFFLINE
+ | KW_MANAGEDLOCATION | KW_MAPJOIN | KW_MATERIALIZED | KW_METADATA |
KW_MINUTE | KW_MONTH | KW_MSCK | KW_NOSCAN | KW_NULLS
| KW_OPTION | KW_OUTPUTDRIVER | KW_OUTPUTFORMAT | KW_OVERWRITE | KW_OWNER
| KW_PARTITIONED | KW_PARTITIONS | KW_PLUS
- | KW_PRINCIPALS | KW_PROTECTION | KW_PURGE | KW_QUERY | KW_QUARTER |
KW_READ | KW_READONLY | KW_REBUILD | KW_RECORDREADER | KW_RECORDWRITER
+ | KW_PRINCIPALS | KW_PURGE | KW_QUERY | KW_QUARTER | KW_READ | KW_REBUILD
| KW_RECORDREADER | KW_RECORDWRITER
| KW_RELOAD | KW_REMOTE | KW_RENAME | KW_REPAIR | KW_REPLACE |
KW_REPLICATION | KW_RESTRICT | KW_REWRITE
| KW_ROLE | KW_ROLES | KW_SCHEMA | KW_SCHEMAS | KW_SECOND | KW_SEMI |
KW_SERDE | KW_SERDEPROPERTIES | KW_SERVER | KW_SETS | KW_SHARED
| KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL |
KW_STATISTICS | KW_STORED | KW_AST
@@ -961,8 +961,8 @@ nonReserved
| KW_NOVALIDATE
| KW_KEY
| KW_MATCHED
- | KW_REPL | KW_DUMP | KW_BATCH | KW_STATUS
- | KW_CACHE | KW_DAYOFWEEK | KW_VIEWS
+ | KW_REPL | KW_DUMP | KW_STATUS
+ | KW_CACHE | KW_VIEWS
| KW_VECTORIZATION
| KW_SUMMARY
| KW_OPERATOR
@@ -971,7 +971,6 @@ nonReserved
| KW_DEBUG
| KW_WAIT
| KW_ZONE
- | KW_TIMESTAMPTZ
| KW_DEFAULT
| KW_REOPTIMIZATION
| KW_EXECUTED | KW_SCHEDULED | KW_CRON | KW_EVERY | KW_AT | KW_EXECUTE
@@ -988,6 +987,16 @@ nonReserved
| KW_TAG
| KW_FAST_FORWARD
| KW_OPTIMIZE
+ | KW_APPLICATION
+ | KW_COMPACT_ID
+ | KW_DATACONNECTOR
+ | KW_DATACONNECTORS
+ | KW_DDL
+ | KW_FORCE
+ | KW_OLDER
+ | KW_PKFK_JOIN
+ | KW_THAN
+ | KW_TIMESTAMPLOCALTZ
;
//The following SQL2011 reserved keywords are used as function name only, but
not as identifiers.
diff --git
a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestNonReservedWords.java
b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestNonReservedWords.java
new file mode 100644
index 00000000000..bcf054924c7
--- /dev/null
+++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestNonReservedWords.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestNonReservedWords {
+ @Parameters(name = "{0}")
+ public static Collection<String> data() {
+ Set<String> keywords = new HashSet<>(HiveParser.getKeywords());
+ Arrays
+ .stream(HiveParser.tokenNames)
+ .filter(token -> token.startsWith("KW_"))
+ .map(token -> token.replaceFirst("^KW_", ""))
+ .forEach(keywords::add);
+ Set<String> reservedWords = new HashSet<>(TestReservedWords.data());
+ return keywords
+ .stream()
+ .filter(keyword -> keyword.matches("[a-zA-Z0-9_]+"))
+ .filter(keyword -> !reservedWords.contains(keyword))
+ .collect(Collectors.toList());
+ }
+
+ private static final Configuration conf = new Configuration();
+ private static final ParseDriver pd = new ParseDriver();
+
+ private final String keyword;
+
+ public TestNonReservedWords(String keyword) {
+ this.keyword = keyword;
+ }
+
+ @Test
+ public void testNonReservedWords() throws Exception {
+ String query = String.format("CREATE TABLE %s (col STRING)", keyword);
+ pd.parse(query, conf);
+ }
+}
diff --git
a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java
b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java
new file mode 100644
index 00000000000..99d6f195d14
--- /dev/null
+++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestReservedWords.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestReservedWords {
+ @Parameters(name = "{0}")
+ public static Collection<String> data() {
+ return Arrays.asList(
+ "ALL",
+ "ALTER",
+ "AND",
+ "ANY",
+ "ARRAY",
+ "AS",
+ "AUTHORIZATION",
+ "BETWEEN",
+ "BIGINT",
+ "BINARY",
+ "BOOLEAN",
+ "BOTH",
+ "BY",
+ "CASE",
+ "CAST",
+ "CHAR",
+ "COLUMN",
+ "COMMIT",
+ "CONF", // Hive specific reserved keyword as of 1.2.0
+ "CONSTRAINT",
+ "CONVERT",
+ "CREATE",
+ "CROSS",
+ "CUBE",
+ "CURRENT",
+ "CURRENT_DATE",
+ "CURRENT_TIMESTAMP",
+ "CURSOR",
+ "DATABASE", // Hive specific reserved keyword as of 1.2.0
+ "DATE",
+ "DECIMAL",
+ "DELETE",
+ "DESCRIBE",
+ "DISTINCT",
+ "DOUBLE",
+ "DROP",
+ "ELSE",
+ "END",
+ "EXCEPT",
+ "EXCHANGE", // Hive specific reserved keyword as of 1.2.0
+ "EXISTS",
+ "EXTENDED", // Hive specific reserved keyword as of 1.2.0
+ "EXTERNAL",
+ "EXTRACT",
+ "FALSE",
+ "FETCH",
+ "FLOAT",
+ "FLOOR",
+ "FOLLOWING", // Hive specific reserved keyword as of 1.2.0
+ "FOR",
+ "FOREIGN",
+ "FROM",
+ "FULL",
+ "FUNCTION",
+ "GRANT",
+ "GROUP",
+ "GROUPING",
+ "HAVING",
+ "IMPORT",
+ "IF", // Hive specific reserved keyword as of 1.2.0
+ "IN",
+ "INNER",
+ "INSERT",
+ "INT",
+ "INTERSECT",
+ "INTERVAL",
+ "INTO",
+ "IS",
+ "JOIN",
+ "LATERAL",
+ "LEADING",
+ "LEFT",
+ "LESS", // Hive specific reserved keyword as of 1.2.0
+ "LIKE",
+ "LOCAL",
+ "MACRO", // Hive specific reserved keyword as of 1.2.0
+ "MAP", // Hive specific reserved keyword as of 1.2.0
+ "MERGE",
+ "MINUS", // HIVE-12765: Hive specific reserved keyword since 2.2.0
+ "MORE", // Hive specific reserved keyword as of 1.2.0
+ "NONE",
+ "NOT",
+ "NULL",
+ "OF",
+ "ON",
+ "ONLY",
+ "OR",
+ "ORDER",
+ "OUT",
+ "OUTER",
+ "OVER",
+ "PARTITION",
+ "PERCENT",
+ "PRECEDING", // Hive specific reserved keyword as of 1.2.0
+ "PRECISION",
+ "PREPARE",
+ "PRESERVE",
+ "PRIMARY",
+ "PROCEDURE",
+ "QUALIFY", // HIVE-25589: Not a part of the ANSI standard, but it
should be reserved
+ "RANGE",
+ "READS",
+ "REAL",
+ "REDUCE", // Hive specific reserved keyword as of 1.2.0
+ "REFERENCES",
+ "REGEXP", // HIVE-11600: Hive specific reserved keyword since 2.0.0
+ "REVOKE",
+ "RIGHT",
+ "RLIKE", // HIVE-11600: Hive specific reserved keyword since 2.0.0
+ "ROLLBACK",
+ "ROLLUP",
+ "ROW",
+ "ROWS",
+ "SELECT",
+ "SET",
+ "SMALLINT",
+ "SOME",
+ "START",
+ "SYNC", // HIVE-17824: Hive specific reserved keyword since 3.0.0
+ "TABLE",
+ "TABLESAMPLE",
+ "THEN",
+ "TIME",
+ "TIMESTAMP",
+ "TO",
+ "TRAILING",
+ "TRANSFORM", // Hive specific reserved keyword as of 1.2.0
+ "TRIGGER",
+ "TRUE",
+ "TRUNCATE",
+ "UNBOUNDED", // Hive specific reserved keyword as of 1.2.0
+ "UNION",
+ "UNIQUE",
+ "UNIQUEJOIN", // Hive specific reserved keyword as of 1.2.0
+ "UPDATE",
+ "USER",
+ "USING",
+ "VALUES",
+ "VARCHAR",
+ "WHEN",
+ "WHERE",
+ "WINDOW",
+ "WITH"
+ );
+ }
+
+ private static final Configuration conf = new Configuration();
+ private static final ParseDriver pd = new ParseDriver();
+
+ private final String keyword;
+
+ public TestReservedWords(String keyword) {
+ this.keyword = keyword;
+ }
+
+ @Test
+ public void testReservedWords() {
+ try {
+ String query = String.format("CREATE TABLE %s (col STRING)", keyword);
+ pd.parse(query, conf);
+ Assert.fail("Expected ParseException");
+ } catch (ParseException e) {
+ if (keyword.equals("IF")) {
+ String expected = "line 1:16 mismatched input '(' expecting NOT near
'IF' in if not exists clause";
+ Assert.assertEquals("Failure didn't match.", expected, e.getMessage());
+ return;
+ }
+ String expected = String.format("line 1:13 cannot recognize input near
'%s' '(' 'col' in table name", keyword);
+ Assert.assertEquals("Failure didn't match.", expected, e.getMessage());
+ }
+ }
+}
