This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c4acfe7 [SPARK-28474][SQL] Hive 0.12 JDBC client can not handle
binary type
c4acfe7 is described below
commit c4acfe7761bde41e9d26c5ab3a670ab86165cf9b
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Thu Aug 8 17:01:25 2019 +0900
[SPARK-28474][SQL] Hive 0.12 JDBC client can not handle binary type
## What changes were proposed in this pull request?
This PR fix Hive 0.12 JDBC client can not handle binary type:
```sql
Connected to: Hive (version 3.0.0-SNAPSHOT)
Driver: Hive (version 0.12.0)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 0.12.0 by Apache Hive
0: jdbc:hive2://localhost:10000> SELECT cast('ABC' as binary);
Error: java.lang.ClassCastException: [B incompatible with java.lang.String
(state=,code=0)
```
Server log:
```
19/08/07 10:10:04 WARN ThriftCLIService: Error fetching results:
java.lang.RuntimeException: java.lang.ClassCastException: [B incompatible
with java.lang.String
at
org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:83)
at
org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:36)
at
org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:63)
at
java.security.AccessController.doPrivileged(AccessController.java:770)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746)
at
org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:59)
at com.sun.proxy.$Proxy26.fetchResults(Unknown Source)
at
org.apache.hive.service.cli.CLIService.fetchResults(CLIService.java:455)
at
org.apache.hive.service.cli.thrift.ThriftCLIService.FetchResults(ThriftCLIService.java:621)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$FetchResults.getResult(TCLIService.java:1553)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$FetchResults.getResult(TCLIService.java:1538)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at
org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:53)
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:310)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:819)
Caused by: java.lang.ClassCastException: [B incompatible with
java.lang.String
at
org.apache.hive.service.cli.ColumnValue.toTColumnValue(ColumnValue.java:198)
at org.apache.hive.service.cli.RowBasedSet.addRow(RowBasedSet.java:60)
at org.apache.hive.service.cli.RowBasedSet.addRow(RowBasedSet.java:32)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$getNextRowSet$1(SparkExecuteStatementOperation.scala:151)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$Lambda$1923.000000009113BFE0.apply(Unknown
Source)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withSchedulerPool(SparkExecuteStatementOperation.scala:299)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.getNextRowSet(SparkExecuteStatementOperation.scala:113)
at
org.apache.hive.service.cli.operation.OperationManager.getOperationNextRowSet(OperationManager.java:220)
at
org.apache.hive.service.cli.session.HiveSessionImpl.fetchResults(HiveSessionImpl.java:785)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:78)
... 18 more
```
## How was this patch tested?
unit tests
Closes #25379 from wangyum/SPARK-28474.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: HyukjinKwon <gurwls...@apache.org>
---
.../SparkThriftServerProtocolVersionsSuite.scala | 14 ++++++++++++--
.../main/java/org/apache/hive/service/cli/ColumnValue.java | 5 ++++-
.../main/java/org/apache/hive/service/cli/ColumnValue.java | 5 ++++-
3 files changed, 20 insertions(+), 4 deletions(-)
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
index e33e42f..604c21f 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
@@ -26,6 +26,9 @@ import org.apache.hive.service.cli.GetInfoType
import org.apache.thrift.protocol.TBinaryProtocol
import org.apache.thrift.transport.TSocket
+import org.apache.spark.sql.catalyst.util.NumberConverter
+import org.apache.spark.unsafe.types.UTF8String
+
class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
override def mode: ServerMode.Value = ServerMode.binary
@@ -215,12 +218,19 @@ class SparkThriftServerProtocolVersionsSuite extends
HiveThriftJdbcTest {
}
}
- // TODO: enable this test case after SPARK-28474
- ignore(s"$version get binary type") {
+ test(s"$version get binary type") {
testExecuteStatementWithProtocolVersion(version, "SELECT cast('ABC' as
binary)") { rs =>
assert(rs.next())
assert(rs.getString(1) === "ABC")
}
+ testExecuteStatementWithProtocolVersion(version, "SELECT cast(49960 as
binary)") { rs =>
+ assert(rs.next())
+ assert(rs.getString(1) ===
UTF8String.fromBytes(NumberConverter.toBinary(49960)).toString)
+ }
+ testExecuteStatementWithProtocolVersion(version, "SELECT cast(null as
binary)") { rs =>
+ assert(rs.next())
+ assert(rs.getString(1) === null)
+ }
}
test(s"$version get boolean type") {
diff --git
a/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/ColumnValue.java
b/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index 40144cf..9e9d6cc 100644
---
a/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++
b/sql/hive-thriftserver/v1.2.1/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -36,6 +36,8 @@ import org.apache.hive.service.cli.thrift.TI32Value;
import org.apache.hive.service.cli.thrift.TI64Value;
import org.apache.hive.service.cli.thrift.TStringValue;
+import org.apache.spark.unsafe.types.UTF8String;
+
/**
* Protocols before HIVE_CLI_SERVICE_PROTOCOL_V6 (used by RowBasedSet)
*
@@ -195,7 +197,8 @@ public class ColumnValue {
case DECIMAL_TYPE:
return stringValue(((HiveDecimal)value));
case BINARY_TYPE:
- return stringValue((String)value);
+ String strVal = value == null ? null :
UTF8String.fromBytes((byte[])value).toString();
+ return stringValue(strVal);
case ARRAY_TYPE:
case MAP_TYPE:
case STRUCT_TYPE:
diff --git
a/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/ColumnValue.java
b/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index 28149e1..12d49b2 100644
---
a/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++
b/sql/hive-thriftserver/v2.3.5/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -37,6 +37,8 @@ import org.apache.hive.service.rpc.thrift.TI32Value;
import org.apache.hive.service.rpc.thrift.TI64Value;
import org.apache.hive.service.rpc.thrift.TStringValue;
+import org.apache.spark.unsafe.types.UTF8String;
+
/**
* Protocols before HIVE_CLI_SERVICE_PROTOCOL_V6 (used by RowBasedSet)
*
@@ -199,7 +201,8 @@ public class ColumnValue {
case DECIMAL_TYPE:
return stringValue((HiveDecimal)value, typeDescriptor);
case BINARY_TYPE:
- return stringValue((String)value);
+ String strVal = value == null ? null :
UTF8String.fromBytes((byte[])value).toString();
+ return stringValue(strVal);
case ARRAY_TYPE:
case MAP_TYPE:
case STRUCT_TYPE:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
