[16/34] ambari git commit: AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query (nitirajrathore)

2017-04-26 Thread jonathanhurley 
AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query 
(nitirajrathore)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b7c023d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b7c023d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b7c023d

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 1b7c023d09827692c47d25bcdecb37669d342035
Parents: 64c9ef8
Author: Nitiraj Singh Rathore 
Authored: Tue Apr 25 17:15:39 2017 +0530
Committer: Nitiraj Singh Rathore 
Committed: Tue Apr 25 17:16:20 2017 +0530

--
 .../generators/InsertFromQueryGenerator.java| 41 +---
 .../uploads/query/InsertFromQueryInput.java | 24 
 .../databases/database/tables/upload-table.js   | 16 +++-
 3 files changed, 66 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
--
diff --git 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
index 8e22fc7..19d4f06 100644
--- 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
+++ 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
@@ -18,7 +18,10 @@
 
 package org.apache.ambari.view.hive20.internal.query.generators;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
 import org.apache.ambari.view.hive20.client.ColumnDescription;
 import org.apache.ambari.view.hive20.exceptions.ServiceException;
 import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
@@ -26,6 +29,10 @@ import 
org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInpu
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+import java.util.LinkedList;
+import java.util.List;
+
 public class InsertFromQueryGenerator implements QueryGenerator{
   protected final static Logger LOG =
   LoggerFactory.getLogger(InsertFromQueryGenerator.class);
@@ -38,12 +45,34 @@ public class InsertFromQueryGenerator implements 
QueryGenerator{
 
   @Override
   public Optional getQuery() throws ServiceException {
-StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
-.append("`").append(insertFromQueryInput.getToTable()).append("`")
-.append(" SELECT ");
+StringBuilder insertQuery = new StringBuilder();
+//Dynamic partition strict mode requires at least one static partition 
column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict
+insertQuery.append("set 
hive.exec.dynamic.partition.mode=nonstrict").append("\n");
+
+insertQuery.append(" FROM 
").append("`").append(insertFromQueryInput.getFromDatabase()).append("`.`")
+.append(insertFromQueryInput.getFromTable()).append("` tempTable");
+
+insertQuery.append(" INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
+.append("`").append(insertFromQueryInput.getToTable()).append("`");
+// PARTITION (partcol1[=val1], partcol2[=val2] ...)
+if(insertFromQueryInput.getPartitionedColumns() != null && 
insertFromQueryInput.getPartitionedColumns().size() > 0){
+  insertQuery.append(" PARTITION ").append("(");
+  
insertQuery.append(Joiner.on(",").join(FluentIterable.from(insertFromQueryInput.getPartitionedColumns()).transform(new
 Function() {
+@Override
+public String apply(ColumnInfo columnInfo) {
+  return "`" + columnInfo.getName() + "`";
+}
+  })));
+  insertQuery.append(" ) ");
+}
+
+insertQuery.append(" SELECT ");
 
+List allColumns = new 
LinkedList<>(insertFromQueryInput.getNormalColumns());
+// this order matters or first normal columns and in the last partitioned 
columns matters.
+allColumns.addAll(insertFromQueryInput.getPartitionedColumns());
 boolean first = true;
-for(ColumnInfo column : insertFromQueryInput.getHeader()){
+for(ColumnInfo column : allColumns){
   String type = column.getType();
   boolean unhex = insertFromQueryInput.getUnhexInsert() && (

ambari git commit: AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query (nitirajrathore)

2017-04-25 Thread nitiraj 
Repository: ambari
Updated Branches:
  refs/heads/trunk 64c9ef8a5 -> 1b7c023d0


AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query 
(nitirajrathore)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b7c023d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b7c023d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b7c023d

Branch: refs/heads/trunk
Commit: 1b7c023d09827692c47d25bcdecb37669d342035
Parents: 64c9ef8
Author: Nitiraj Singh Rathore 
Authored: Tue Apr 25 17:15:39 2017 +0530
Committer: Nitiraj Singh Rathore 
Committed: Tue Apr 25 17:16:20 2017 +0530

--
 .../generators/InsertFromQueryGenerator.java| 41 +---
 .../uploads/query/InsertFromQueryInput.java | 24 
 .../databases/database/tables/upload-table.js   | 16 +++-
 3 files changed, 66 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
--
diff --git 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
index 8e22fc7..19d4f06 100644
--- 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
+++ 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
@@ -18,7 +18,10 @@
 
 package org.apache.ambari.view.hive20.internal.query.generators;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
 import org.apache.ambari.view.hive20.client.ColumnDescription;
 import org.apache.ambari.view.hive20.exceptions.ServiceException;
 import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
@@ -26,6 +29,10 @@ import 
org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInpu
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+import java.util.LinkedList;
+import java.util.List;
+
 public class InsertFromQueryGenerator implements QueryGenerator{
   protected final static Logger LOG =
   LoggerFactory.getLogger(InsertFromQueryGenerator.class);
@@ -38,12 +45,34 @@ public class InsertFromQueryGenerator implements 
QueryGenerator{
 
   @Override
   public Optional getQuery() throws ServiceException {
-StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
-.append("`").append(insertFromQueryInput.getToTable()).append("`")
-.append(" SELECT ");
+StringBuilder insertQuery = new StringBuilder();
+//Dynamic partition strict mode requires at least one static partition 
column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict
+insertQuery.append("set 
hive.exec.dynamic.partition.mode=nonstrict").append("\n");
+
+insertQuery.append(" FROM 
").append("`").append(insertFromQueryInput.getFromDatabase()).append("`.`")
+.append(insertFromQueryInput.getFromTable()).append("` tempTable");
+
+insertQuery.append(" INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
+.append("`").append(insertFromQueryInput.getToTable()).append("`");
+// PARTITION (partcol1[=val1], partcol2[=val2] ...)
+if(insertFromQueryInput.getPartitionedColumns() != null && 
insertFromQueryInput.getPartitionedColumns().size() > 0){
+  insertQuery.append(" PARTITION ").append("(");
+  
insertQuery.append(Joiner.on(",").join(FluentIterable.from(insertFromQueryInput.getPartitionedColumns()).transform(new
 Function() {
+@Override
+public String apply(ColumnInfo columnInfo) {
+  return "`" + columnInfo.getName() + "`";
+}
+  })));
+  insertQuery.append(" ) ");
+}
+
+insertQuery.append(" SELECT ");
 
+List allColumns = new 
LinkedList<>(insertFromQueryInput.getNormalColumns());
+// this order matters or first normal columns and in the last partitioned 
columns matters.
+allColumns.addAll(insertFromQueryInput.getPartitionedColumns());
 boolean first = true;
-for(ColumnInfo column : insertFromQueryInput.getHeader()){
+for(ColumnInfo column : allColumns){
   String type = column.getType();
   boolean

ambari git commit: AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query (nitirajrathore)

2017-04-25 Thread nitiraj 
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 a76b3f061 -> 7df0518f2


AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query 
(nitirajrathore)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7df0518f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7df0518f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7df0518f

Branch: refs/heads/branch-2.5
Commit: 7df0518f2c54c9c27b435f9e477c2231832376bc
Parents: a76b3f0
Author: Nitiraj Singh Rathore 
Authored: Tue Apr 25 17:15:39 2017 +0530
Committer: Nitiraj Singh Rathore 
Committed: Tue Apr 25 17:15:39 2017 +0530

--
 .../generators/InsertFromQueryGenerator.java| 41 +---
 .../uploads/query/InsertFromQueryInput.java | 24 
 .../databases/database/tables/upload-table.js   | 16 +++-
 3 files changed, 66 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/7df0518f/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
--
diff --git 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
index 8e22fc7..19d4f06 100644
--- 
a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
+++ 
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
@@ -18,7 +18,10 @@
 
 package org.apache.ambari.view.hive20.internal.query.generators;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
 import org.apache.ambari.view.hive20.client.ColumnDescription;
 import org.apache.ambari.view.hive20.exceptions.ServiceException;
 import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
@@ -26,6 +29,10 @@ import 
org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInpu
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+import java.util.LinkedList;
+import java.util.List;
+
 public class InsertFromQueryGenerator implements QueryGenerator{
   protected final static Logger LOG =
   LoggerFactory.getLogger(InsertFromQueryGenerator.class);
@@ -38,12 +45,34 @@ public class InsertFromQueryGenerator implements 
QueryGenerator{
 
   @Override
   public Optional getQuery() throws ServiceException {
-StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
-.append("`").append(insertFromQueryInput.getToTable()).append("`")
-.append(" SELECT ");
+StringBuilder insertQuery = new StringBuilder();
+//Dynamic partition strict mode requires at least one static partition 
column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict
+insertQuery.append("set 
hive.exec.dynamic.partition.mode=nonstrict").append("\n");
+
+insertQuery.append(" FROM 
").append("`").append(insertFromQueryInput.getFromDatabase()).append("`.`")
+.append(insertFromQueryInput.getFromTable()).append("` tempTable");
+
+insertQuery.append(" INSERT INTO TABLE 
`").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
+.append("`").append(insertFromQueryInput.getToTable()).append("`");
+// PARTITION (partcol1[=val1], partcol2[=val2] ...)
+if(insertFromQueryInput.getPartitionedColumns() != null && 
insertFromQueryInput.getPartitionedColumns().size() > 0){
+  insertQuery.append(" PARTITION ").append("(");
+  
insertQuery.append(Joiner.on(",").join(FluentIterable.from(insertFromQueryInput.getPartitionedColumns()).transform(new
 Function() {
+@Override
+public String apply(ColumnInfo columnInfo) {
+  return "`" + columnInfo.getName() + "`";
+}
+  })));
+  insertQuery.append(" ) ");
+}
+
+insertQuery.append(" SELECT ");
 
+List allColumns = new 
LinkedList<>(insertFromQueryInput.getNormalColumns());
+// this order matters or first normal columns and in the last partitioned 
columns matters.
+allColumns.addAll(insertFromQueryInput.getPartitionedColumns());
 boolean first = true;
-for(ColumnInfo column : insertFromQueryInput.getHeader()){
+for(ColumnInfo column : allColumns){
   String type = column.getType();