Repository: phoenix Updated Branches: refs/heads/master e906841fb -> b32699505
PHOENIX-3544 Fix UNSIGNED_DATE and UNSIGNED_TIME type in phoenix-spark Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b3269950 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b3269950 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b3269950 Branch: refs/heads/master Commit: b32699505d18b5b93b4072e0b146df6b8ddaa7fb Parents: e906841 Author: Ankit Singhal <ankitsingha...@gmail.com> Authored: Mon Dec 26 16:54:13 2016 +0530 Committer: Ankit Singhal <ankitsingha...@gmail.com> Committed: Mon Dec 26 16:54:13 2016 +0530 ---------------------------------------------------------------------- phoenix-spark/src/it/resources/globalSetup.sql | 5 ++++- .../it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala | 8 ++++++++ .../src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala | 5 ++--- 3 files changed, 14 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/b3269950/phoenix-spark/src/it/resources/globalSetup.sql ---------------------------------------------------------------------- diff --git a/phoenix-spark/src/it/resources/globalSetup.sql b/phoenix-spark/src/it/resources/globalSetup.sql index 72f8620..28eb0f7 100644 --- a/phoenix-spark/src/it/resources/globalSetup.sql +++ b/phoenix-spark/src/it/resources/globalSetup.sql @@ -57,4 +57,7 @@ UPSERT INTO "small" VALUES ('key1', 'foo', 10000) UPSERT INTO "small" VALUES ('key2', 'bar', 20000) UPSERT INTO "small" VALUES ('key3', 'xyz', 30000) -CREATE TABLE MULTITENANT_TEST_TABLE (TENANT_ID VARCHAR NOT NULL, ORGANIZATION_ID VARCHAR, GLOBAL_COL1 VARCHAR CONSTRAINT pk PRIMARY KEY (TENANT_ID, ORGANIZATION_ID)) MULTI_TENANT=true \ No newline at end of file +CREATE TABLE MULTITENANT_TEST_TABLE (TENANT_ID VARCHAR NOT NULL, ORGANIZATION_ID VARCHAR, GLOBAL_COL1 VARCHAR CONSTRAINT pk PRIMARY KEY (TENANT_ID, ORGANIZATION_ID)) MULTI_TENANT=true +CREATE TABLE IF NOT EXISTS GIGANTIC_TABLE (ID INTEGER PRIMARY KEY,unsig_id UNSIGNED_INT,big_id BIGINT,unsig_long_id UNSIGNED_LONG,tiny_id TINYINT,unsig_tiny_id UNSIGNED_TINYINT,small_id SMALLINT,unsig_small_id UNSIGNED_SMALLINT,float_id FLOAT,unsig_float_id UNSIGNED_FLOAT,double_id DOUBLE,unsig_double_id UNSIGNED_DOUBLE,decimal_id DECIMAL,boolean_id BOOLEAN,time_id TIME,date_id DATE,timestamp_id TIMESTAMP,unsig_time_id UNSIGNED_TIME,unsig_date_id UNSIGNED_DATE,unsig_timestamp_id UNSIGNED_TIMESTAMP,varchar_id VARCHAR (30),char_id CHAR (30),binary_id BINARY (100),varbinary_id VARBINARY (100)) + CREATE TABLE IF NOT EXISTS OUTPUT_GIGANTIC_TABLE (ID INTEGER PRIMARY KEY,unsig_id UNSIGNED_INT,big_id BIGINT,unsig_long_id UNSIGNED_LONG,tiny_id TINYINT,unsig_tiny_id UNSIGNED_TINYINT,small_id SMALLINT,unsig_small_id UNSIGNED_SMALLINT,float_id FLOAT,unsig_float_id UNSIGNED_FLOAT,double_id DOUBLE,unsig_double_id UNSIGNED_DOUBLE,decimal_id DECIMAL,boolean_id BOOLEAN,time_id TIME,date_id DATE,timestamp_id TIMESTAMP,unsig_time_id UNSIGNED_TIME,unsig_date_id UNSIGNED_DATE,unsig_timestamp_id UNSIGNED_TIMESTAMP,varchar_id VARCHAR (30),char_id CHAR (30),binary_id BINARY (100),varbinary_id VARBINARY (100)) + upsert into GIGANTIC_TABLE values(0,2,3,4,-5,6,7,8,9.3,10.4,11.5,12.6,13.7,true,CURRENT_TIME(),CURRENT_DATE(),CURRENT_TIME(),CURRENT_TIME(),CURRENT_DATE(),CURRENT_TIME(),'This is random textA','a','a','a') http://git-wip-us.apache.org/repos/asf/phoenix/blob/b3269950/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala ---------------------------------------------------------------------- diff --git a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala index dbcc4f1..9def354 100644 --- a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala +++ b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala @@ -632,5 +632,13 @@ class PhoenixSparkIT extends AbstractPhoenixSparkIT { val epoch = new Date().getTime assert(Math.abs(epoch - time) < 86400000) } + test("can read all Phoenix data types") { + val sqlContext = new SQLContext(sc) + val df = sqlContext.load("org.apache.phoenix.spark", Map("table" -> "GIGANTIC_TABLE", + "zkUrl" -> quorumAddress)) + df.save("org.apache.phoenix.spark",SaveMode.Overwrite, Map("table" -> "OUTPUT_GIGANTIC_TABLE", + "zkUrl" -> quorumAddress)) + df.count() shouldEqual 1 + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b3269950/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala ---------------------------------------------------------------------- diff --git a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala index 204a7ef..63289a0 100644 --- a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala +++ b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala @@ -129,11 +129,10 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String], // Create a sequence of column data val rowSeq = columns.map { case (name, sqlType) => val res = pr.resultMap(name) - // Special handling for data types - if (dateAsTimestamp && sqlType == 91) { // 91 is the defined type for Date + if (dateAsTimestamp && (sqlType == 91 || sqlType == 19)) { // 91 is the defined type for Date and 19 for UNSIGNED_DATE new java.sql.Timestamp(res.asInstanceOf[java.sql.Date].getTime) - } else if (sqlType == 92) { // 92 is the defined type for Time + } else if (sqlType == 92 || sqlType == 18) { // 92 is the defined type for Time and 18 for UNSIGNED_TIME new java.sql.Timestamp(res.asInstanceOf[java.sql.Time].getTime) } else { res