AMBARI-20181 : hive view 2.0 : added back ticks in column names to support all unicode chars (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d562216e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d562216e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d562216e Branch: refs/heads/branch-feature-AMBARI-12556 Commit: d562216ed7784fd5898668b17ddf80abdbf642b1 Parents: a749f97 Author: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Authored: Wed Mar 1 23:39:55 2017 +0530 Committer: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Committed: Wed Mar 1 23:39:55 2017 +0530 ---------------------------------------------------------------------- .../view/hive20/internal/dto/ColumnInfo.java | 19 ++++-- .../generators/CreateTableQueryGenerator.java | 9 ++- .../generators/InsertFromQueryGenerator.java | 6 +- .../CreateTableQueryGeneratorTest.java | 11 +++- .../InsertFromQueryGeneratorSpecTest.groovy | 66 ++++++++++++++++++++ 5 files changed, 99 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d562216e/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnInfo.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnInfo.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnInfo.java index 9f179d1..e5cd540 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnInfo.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnInfo.java @@ -45,14 +45,23 @@ public class ColumnInfo { } public ColumnInfo(String name, String type, String comment) { - this.name = name; - this.type = type; - this.comment = comment; + this(name, type, null, null, comment); + } + + public ColumnInfo(String name, String type, Integer precision, String comment) { + this(name, type, precision, null, comment); + } + + public ColumnInfo(String name, String type, Integer precision, Integer scale) { + this(name, type, precision, scale, null); + } + + public ColumnInfo(String name, String type, Integer precision) { + this(name, type, precision, null, null); } public ColumnInfo(String name, String type) { - this.name = name; - this.type = type; + this(name, type, null, null, null); } public String getName() { http://git-wip-us.apache.org/repos/asf/ambari/blob/d562216e/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGenerator.java index eab3a4b..774075d 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGenerator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGenerator.java @@ -48,13 +48,16 @@ public class CreateTableQueryGenerator implements QueryGenerator{ public Optional<String> getQuery(){ StringBuffer query = new StringBuffer(); query.append("CREATE TABLE "); - query.append(tableMeta.getDatabase()).append("."); - query.append(tableMeta.getTable()).append(" "); + query.append("`").append(tableMeta.getDatabase()).append("`").append("."); + query.append("`").append(tableMeta.getTable()).append("`").append(" "); query.append("(").append(getColumnQuery(tableMeta.getColumns())).append(") "); if(null != tableMeta.getDetailedInfo() && null != tableMeta.getDetailedInfo().getParameters()){ String tableComment = tableMeta.getDetailedInfo().getParameters().get(COMMENT); if(!Strings.isNullOrEmpty(tableComment)){ - query.append(" COMMENT ").append(tableComment); + tableComment = tableMeta.getDetailedInfo().getParameters().get(COMMENT.toLowerCase()); + if(!Strings.isNullOrEmpty(tableComment)) { + query.append(" COMMENT ").append(tableComment); + } } } if(null != tableMeta.getPartitionInfo() ) { http://git-wip-us.apache.org/repos/asf/ambari/blob/d562216e/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java index b544100..8e22fc7 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java @@ -38,8 +38,8 @@ public class InsertFromQueryGenerator implements QueryGenerator{ @Override public Optional<String> getQuery() throws ServiceException { - StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append("`.`") - .append(insertFromQueryInput.getToTable()).append("`") + StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append('`').append(".") + .append("`").append(insertFromQueryInput.getToTable()).append("`") .append(" SELECT "); boolean first = true; @@ -59,7 +59,7 @@ public class InsertFromQueryGenerator implements QueryGenerator{ insertQuery.append("UNHEX("); } - insertQuery.append(column.getName()); + insertQuery.append('`').append(column.getName()).append('`'); if(unhex) { insertQuery.append(")"); http://git-wip-us.apache.org/repos/asf/ambari/blob/d562216e/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGeneratorTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGeneratorTest.java b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGeneratorTest.java index f5a9778..ebb0834 100644 --- a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGeneratorTest.java +++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/CreateTableQueryGeneratorTest.java @@ -30,6 +30,14 @@ public class CreateTableQueryGeneratorTest { @Test public void testGetQuery() throws Exception { + String createTableQuery = "CREATE TABLE `d1`.`t2` (`col_name1` string COMMENT 'col_name1 comment'," + + "`col_name2` decimal(10,2) COMMENT 'col_name2 comment') PARTITIONED BY ( `col_name4` char(1) COMMENT 'col_name4 comment'," + + "`col_name3` string COMMENT 'col_name3 comment') CLUSTERED BY (col_name1, col_name2) SORTED BY (col_name1 ASC,col_name2 DESC)" + + " INTO 5 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ESCAPED BY '\\\\' STORED AS INPUTFORMAT " + + "'org.apache.hadoop.mapred.SequenceFileInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat' " + + "LOCATION 'hdfs://c6401.ambari.apache.org:8020/user/hive/tables/d1/t1' TBLPROPERTIES ('immutable'='false'," + + "'orc.compress'='SNAPPY','transient_lastDdlTime'='1481520077','NO_AUTO_COMPACTION'='true','comment'='table t1 comment'," + + "'SORTBUCKETCOLSPREFIX'='TRUE')"; String json = "{\n" + "\t\"id\": \"d1/t2\",\n" + "\t\"database\": \"d1\",\n" + @@ -43,7 +51,6 @@ public class CreateTableQueryGeneratorTest { "\t\t\"type\": \"decimal(10,2)\",\n" + "\t\t\"comment\": \"col_name2 comment\"\n" + "\t}],\n" + - "\t\"ddl\": \"CREATE TABLE `t2`(\\n `col_name1` string COMMENT \\u0027col_name1 comment\\u0027, \\n `col_name2` decimal(10,2) COMMENT \\u0027col_name2 comment\\u0027)\\nCOMMENT \\u0027table t1 comment\\u0027\\nPARTITIONED BY ( \\n `col_name3` string COMMENT \\u0027col_name3 comment\\u0027, \\n `col_name4` char(1) COMMENT \\u0027col_name4 comment\\u0027)\\nCLUSTERED BY ( \\n col_name1, \\n col_name2) \\nSORTED BY ( \\n col_name1 ASC, \\n col_name2 DESC) \\nINTO 5 BUCKETS\\nROW FORMAT DELIMITED \\n FIELDS TERMINATED BY \\u0027,\\u0027 \\nWITH SERDEPROPERTIES ( \\n \\u0027escape.delim\\u0027\\u003d\\u0027\\\\\\\\\\u0027) \\nSTORED AS INPUTFORMAT \\n \\u0027org.apache.hadoop.mapred.SequenceFileInputFormat\\u0027 \\nOUTPUTFORMAT \\n \\u0027org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat\\u0027\\nLOCATION\\n \\u0027hdfs://c6401.ambari.apache.org:8020/user/hive/tables/d1/t1\\u0027\\nTBLPROPERTIES (\\n \\u0027NO_AUTO_COMPACTION\\u0027\\u003d\\u0027true\\u0027, \\n \\u0027immutable\\u0027\\u003d\\u0027false\\u0027, \\n \\u0027orc.compress\\u0027\\u003d\\u0027SNAPPY\\u0027, \\n \\u0027transient_lastDdlTime\\u0027\\u003d\\u00271481520077\\u0027)\\n\",\n" + "\t\"partitionInfo\": {\n" + "\t\t\"columns\": [{\n" + "\t\t\t\"name\": \"col_name4\",\n" + @@ -97,5 +104,7 @@ public class CreateTableQueryGeneratorTest { Optional<String> createQuery = new CreateTableQueryGenerator(tableMeta).getQuery(); LOG.info("createQuery : {}", createQuery); Assert.assertTrue(createQuery.isPresent()); + + Assert.assertEquals( "incorrect create table query.", createTableQuery, createQuery.get()); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d562216e/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGeneratorSpecTest.groovy ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGeneratorSpecTest.groovy b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGeneratorSpecTest.groovy new file mode 100644 index 0000000..dfdf9df --- /dev/null +++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGeneratorSpecTest.groovy @@ -0,0 +1,66 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ambari.view.hive20.internal.query.generators + +import com.google.common.base.Optional +import org.apache.ambari.view.hive20.exceptions.ServiceException +import org.apache.ambari.view.hive20.internal.dto.ColumnInfo +import org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInput +import spock.lang.Specification + +class InsertFromQueryGeneratorSpecTest extends Specification { + def "insert from without unhexing"() { + setup: + List<ColumnInfo> colInfos = Arrays.asList(new ColumnInfo("col1", "STRING"), new ColumnInfo("col2", "INT"), new ColumnInfo("col3", "VARCHAR", 255), + new ColumnInfo("col4", "CHAR", 25)) + InsertFromQueryInput insertFromQueryInput = new InsertFromQueryInput("d1", "t1", "d2", "t2", colInfos, false) + InsertFromQueryGenerator generator = new InsertFromQueryGenerator(insertFromQueryInput); + + when: + Optional<String> query = generator.getQuery() + + then: + query.isPresent() + + when: + String queryStr = query.get(); + + then: + queryStr == "INSERT INTO TABLE `d2`.`t2` SELECT `col1`, `col2`, `col3`, `col4` FROM `d1.t1` ;" + } + + def "insert from with unhexing"() { + setup: + List<ColumnInfo> colInfos = Arrays.asList(new ColumnInfo("col1", "STRING"), new ColumnInfo("col2", "INT"), new ColumnInfo("col3", "VARCHAR", 255), + new ColumnInfo("col4", "CHAR", 25)) + InsertFromQueryInput insertFromQueryInput = new InsertFromQueryInput("d1", "t1", "d2", "t2", colInfos, true) + InsertFromQueryGenerator generator = new InsertFromQueryGenerator(insertFromQueryInput); + + when: + Optional<String> query = generator.getQuery() + + then: + query.isPresent() + + when: + String queryStr = query.get(); + + then: + queryStr == "INSERT INTO TABLE `d2`.`t2` SELECT UNHEX(`col1`), `col2`, UNHEX(`col3`), UNHEX(`col4`) FROM `d1.t1` ;" + } +}
