spark git commit: [SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

2017-10-14 Thread lixiao 
Repository: spark
Updated Branches:
  refs/heads/branch-2.1 920372a19 -> eb00037a7


[SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

## What changes were proposed in this pull request?

When fixing schema field names using escape characters with 
`addReferenceMinorObj()` at 
[SPARK-18952](https://issues.apache.org/jira/browse/SPARK-18952) (#16361), 
double-quotes around the names were remained and the names become something 
like `"((java.lang.String) references[1])"`.

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[1])", org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[2])", org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

We should remove the double-quotes to refer the values in `references` properly:

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[1]), 
org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[2]), 
org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

## How was this patch tested?

Existing tests.

Author: Takuya UESHIN 

Closes #19491 from ueshin/issues/SPARK-22273.

(cherry picked from commit e0503a7223410289d01bc4b20da3a451730577da)
Signed-off-by: gatorsmile 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb00037a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb00037a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb00037a

Branch: refs/heads/branch-2.1
Commit: eb00037a70614daaf94a3ae59d3126ba696da3e2
Parents: 920372a
Author: Takuya UESHIN 
Authored: Fri Oct 13 23:24:36 2017 -0700
Committer: gatorsmile 
Committed: Fri Oct 13 23:25:21 2017 -0700

--
 .../sql/execution/aggregate/RowBasedHashMapGenerator.scala   | 8 
 .../sql/execution/aggregate/VectorizedHashMapGenerator.scala | 8 
 2 files changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/eb00037a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
--
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
index 1b6e6d2..ffb0c6c 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
@@ -50,10 +50,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")
 
@@ -63,10 +63,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")

spark git commit: [SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

2017-10-14 Thread lixiao 
Repository: spark
Updated Branches:
  refs/heads/branch-2.2 30d5c9fd8 -> acbad83ec


[SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

## What changes were proposed in this pull request?

When fixing schema field names using escape characters with 
`addReferenceMinorObj()` at 
[SPARK-18952](https://issues.apache.org/jira/browse/SPARK-18952) (#16361), 
double-quotes around the names were remained and the names become something 
like `"((java.lang.String) references[1])"`.

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[1])", org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[2])", org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

We should remove the double-quotes to refer the values in `references` properly:

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[1]), 
org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[2]), 
org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

## How was this patch tested?

Existing tests.

Author: Takuya UESHIN 

Closes #19491 from ueshin/issues/SPARK-22273.

(cherry picked from commit e0503a7223410289d01bc4b20da3a451730577da)
Signed-off-by: gatorsmile 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/acbad83e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/acbad83e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/acbad83e

Branch: refs/heads/branch-2.2
Commit: acbad83ecd7d3e19f0870021a9dd342a85897ae4
Parents: 30d5c9f
Author: Takuya UESHIN 
Authored: Fri Oct 13 23:24:36 2017 -0700
Committer: gatorsmile 
Committed: Fri Oct 13 23:24:49 2017 -0700

--
 .../sql/execution/aggregate/RowBasedHashMapGenerator.scala   | 8 
 .../sql/execution/aggregate/VectorizedHashMapGenerator.scala | 8 
 2 files changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/acbad83e/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
--
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
index 9316ebc..3718424 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
@@ -50,10 +50,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceMinorObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")
 
@@ -63,10 +63,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceMinorObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")

spark git commit: [SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

2017-10-14 Thread lixiao 
Repository: spark
Updated Branches:
  refs/heads/master e3536406e -> e0503a722


[SPARK-22273][SQL] Fix key/value schema field names in HashMapGenerators.

## What changes were proposed in this pull request?

When fixing schema field names using escape characters with 
`addReferenceMinorObj()` at 
[SPARK-18952](https://issues.apache.org/jira/browse/SPARK-18952) (#16361), 
double-quotes around the names were remained and the names become something 
like `"((java.lang.String) references[1])"`.

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[1])", org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add("((java.lang.String) 
references[2])", org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

We should remove the double-quotes to refer the values in `references` properly:

```java
/* 055 */ private int maxSteps = 2;
/* 056 */ private int numRows = 0;
/* 057 */ private org.apache.spark.sql.types.StructType keySchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[1]), 
org.apache.spark.sql.types.DataTypes.StringType);
/* 058 */ private org.apache.spark.sql.types.StructType valueSchema = new 
org.apache.spark.sql.types.StructType().add(((java.lang.String) references[2]), 
org.apache.spark.sql.types.DataTypes.LongType);
/* 059 */ private Object emptyVBase;
```

## How was this patch tested?

Existing tests.

Author: Takuya UESHIN 

Closes #19491 from ueshin/issues/SPARK-22273.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0503a72
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0503a72
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0503a72

Branch: refs/heads/master
Commit: e0503a7223410289d01bc4b20da3a451730577da
Parents: e353640
Author: Takuya UESHIN 
Authored: Fri Oct 13 23:24:36 2017 -0700
Committer: gatorsmile 
Committed: Fri Oct 13 23:24:36 2017 -0700

--
 .../sql/execution/aggregate/RowBasedHashMapGenerator.scala   | 8 
 .../sql/execution/aggregate/VectorizedHashMapGenerator.scala | 8 
 2 files changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/e0503a72/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
--
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
index 9316ebc..3718424 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/RowBasedHashMapGenerator.scala
@@ -50,10 +50,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceMinorObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")
 
@@ -63,10 +63,10 @@ class RowBasedHashMapGenerator(
   val keyName = ctx.addReferenceMinorObj(key.name)
   key.dataType match {
 case d: DecimalType =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.createDecimalType(
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.createDecimalType(
   |${d.precision}, ${d.scale}))""".stripMargin
 case _ =>
-  s""".add("$keyName", 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
+  s""".add($keyName, 
org.apache.spark.sql.types.DataTypes.${key.dataType})"""
   }
 }.mkString("\n").concat(";")
 

http://git-wip-us.apache.org/repos/asf/spark/blob/e0503a72/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/VectorizedHashMapGenerator.scala
--
diff --git