AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b7c023d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b7c023d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b7c023d Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 1b7c023d09827692c47d25bcdecb37669d342035 Parents: 64c9ef8 Author: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Authored: Tue Apr 25 17:15:39 2017 +0530 Committer: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Committed: Tue Apr 25 17:16:20 2017 +0530 ---------------------------------------------------------------------- .../generators/InsertFromQueryGenerator.java | 41 +++++++++++++++++--- .../uploads/query/InsertFromQueryInput.java | 24 ++++++++---- .../databases/database/tables/upload-table.js | 16 +++++++- 3 files changed, 66 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java index 8e22fc7..19d4f06 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java @@ -18,7 +18,10 @@ package org.apache.ambari.view.hive20.internal.query.generators; +import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; import org.apache.ambari.view.hive20.client.ColumnDescription; import org.apache.ambari.view.hive20.exceptions.ServiceException; import org.apache.ambari.view.hive20.internal.dto.ColumnInfo; @@ -26,6 +29,10 @@ import org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInpu import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import java.util.LinkedList; +import java.util.List; + public class InsertFromQueryGenerator implements QueryGenerator{ protected final static Logger LOG = LoggerFactory.getLogger(InsertFromQueryGenerator.class); @@ -38,12 +45,34 @@ public class InsertFromQueryGenerator implements QueryGenerator{ @Override public Optional<String> getQuery() throws ServiceException { - StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append('`').append(".") - .append("`").append(insertFromQueryInput.getToTable()).append("`") - .append(" SELECT "); + StringBuilder insertQuery = new StringBuilder(); + //Dynamic partition strict mode requires at least one static partition column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict + insertQuery.append("set hive.exec.dynamic.partition.mode=nonstrict").append("\n"); + + insertQuery.append(" FROM ").append("`").append(insertFromQueryInput.getFromDatabase()).append("`.`") + .append(insertFromQueryInput.getFromTable()).append("` tempTable"); + + insertQuery.append(" INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append('`').append(".") + .append("`").append(insertFromQueryInput.getToTable()).append("`"); + // PARTITION (partcol1[=val1], partcol2[=val2] ...) + if(insertFromQueryInput.getPartitionedColumns() != null && insertFromQueryInput.getPartitionedColumns().size() > 0){ + insertQuery.append(" PARTITION ").append("("); + insertQuery.append(Joiner.on(",").join(FluentIterable.from(insertFromQueryInput.getPartitionedColumns()).transform(new Function<ColumnInfo, String>() { + @Override + public String apply(ColumnInfo columnInfo) { + return "`" + columnInfo.getName() + "`"; + } + }))); + insertQuery.append(" ) "); + } + + insertQuery.append(" SELECT "); + List<ColumnInfo> allColumns = new LinkedList<>(insertFromQueryInput.getNormalColumns()); + // this order matters or first normal columns and in the last partitioned columns matters. + allColumns.addAll(insertFromQueryInput.getPartitionedColumns()); boolean first = true; - for(ColumnInfo column : insertFromQueryInput.getHeader()){ + for(ColumnInfo column : allColumns){ String type = column.getType(); boolean unhex = insertFromQueryInput.getUnhexInsert() && ( ColumnDescription.DataTypes.STRING.toString().equals(type) @@ -59,6 +88,7 @@ public class InsertFromQueryGenerator implements QueryGenerator{ insertQuery.append("UNHEX("); } + insertQuery.append("tempTable."); insertQuery.append('`').append(column.getName()).append('`'); if(unhex) { @@ -68,8 +98,7 @@ public class InsertFromQueryGenerator implements QueryGenerator{ first = false; } - insertQuery.append(" FROM ").append("`").append(insertFromQueryInput.getFromDatabase()).append(".") - .append(insertFromQueryInput.getFromTable()).append("` ").append(";"); + insertQuery.append(";"); String query = insertQuery.toString(); LOG.info("Insert From Query : {}", query); return Optional.of(query); http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java index 4ff61b4..b74ba9b 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java @@ -28,28 +28,38 @@ public class InsertFromQueryInput { private String fromTable; private String toDatabase; private String toTable; - private List<ColumnInfo> header; + private List<ColumnInfo> partitionedColumns; + private List<ColumnInfo> normalColumns; private Boolean unhexInsert = Boolean.FALSE; public InsertFromQueryInput() { } public InsertFromQueryInput(String fromDatabase, String fromTable, String toDatabase, String toTable, - List<ColumnInfo> header, Boolean unhexInsert) { + List<ColumnInfo> partitionedColumns, List<ColumnInfo> normalColumns, Boolean unhexInsert) { this.fromDatabase = fromDatabase; this.fromTable = fromTable; this.toDatabase = toDatabase; this.toTable = toTable; - this.header = header; + this.partitionedColumns = partitionedColumns; + this.normalColumns = normalColumns; this.unhexInsert = unhexInsert; } - public List<ColumnInfo> getHeader() { - return header; + public List<ColumnInfo> getPartitionedColumns() { + return partitionedColumns; } - public void setHeader(List<ColumnInfo> header) { - this.header = header; + public void setPartitionedColumns(List<ColumnInfo> partitionedColumns) { + this.partitionedColumns = partitionedColumns; + } + + public List<ColumnInfo> getNormalColumns() { + return normalColumns; + } + + public void setNormalColumns(List<ColumnInfo> normalColumns) { + this.normalColumns = normalColumns; } public Boolean getUnhexInsert() { http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js index 5fee140..f47d820 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js +++ b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js @@ -279,6 +279,7 @@ export default NewTable.extend(UILoggerMixin, { column.set("scale"); column.set("precision"); } + column.set("isPartitioned", false); // remove all partitioning information from temp table. return column; }); @@ -421,7 +422,17 @@ export default NewTable.extend(UILoggerMixin, { console.log("insertIntoTable"); this.pushUploadProgressInfos(this.formatMessage('hive.messages.startingToInsertRows')); - let headers = tableData.get("tableMeta").columns.map(function(column){ + let partitionedColumns = tableData.get("tableMeta").columns.filter(function(column){ + return column.isPartitioned; + }).map(function(column){ + var header = JSON.parse(JSON.stringify(column)); + header.type = column.type.label; + return header; + }); + + let normalColumns = tableData.get("tableMeta").columns.filter(function(column){ + return !column.isPartitioned; + }).map(function(column){ var header = JSON.parse(JSON.stringify(column)); header.type = column.type.label; return header; @@ -432,7 +443,8 @@ export default NewTable.extend(UILoggerMixin, { "fromTable": tableData.get("tempTableMeta").name, "toDatabase": tableData.get("database"), "toTable": tableData.get("tableMeta").name, - "header": headers, + "partitionedColumns": partitionedColumns, + "normalColumns": normalColumns, "unhexInsert": tableData.fileFormatInfo.containsEndlines }); },