Repository: incubator-carbondata Updated Branches: refs/heads/master fc28fef2c -> 6fe4f96e1
Fixed Data load fail if dataframe is created with LONG datatype column Fixed review comment Moved test case to SparkDatasourceSuite Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a82a1270 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a82a1270 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a82a1270 Branch: refs/heads/master Commit: a82a1270afb9cafd4538c4922eaf89cee20eb80a Parents: fc28fef Author: ravipesala <ravi.pes...@gmail.com> Authored: Thu Nov 10 22:16:59 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Mon Nov 14 16:33:53 2016 +0800 ---------------------------------------------------------------------- .../spark/CarbonDataFrameWriter.scala | 18 +++++++++--------- integration/spark/src/test/resources/test.json | 3 +++ .../dataload/SparkDatasourceSuite.scala | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a82a1270/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala index 2c5fb16..c2b2190 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala @@ -136,15 +136,15 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) extends Logging { private def convertToCarbonType(sparkType: DataType): String = { sparkType match { - case StringType => CarbonType.STRING.name - case IntegerType => CarbonType.INT.name - case ByteType => CarbonType.INT.name - case ShortType => CarbonType.SHORT.name - case LongType => CarbonType.LONG.name - case FloatType => CarbonType.DOUBLE.name - case DoubleType => CarbonType.DOUBLE.name - case BooleanType => CarbonType.DOUBLE.name - case TimestampType => CarbonType.TIMESTAMP.name + case StringType => CarbonType.STRING.getName + case IntegerType => CarbonType.INT.getName + case ByteType => CarbonType.INT.getName + case ShortType => CarbonType.SHORT.getName + case LongType => CarbonType.LONG.getName + case FloatType => CarbonType.DOUBLE.getName + case DoubleType => CarbonType.DOUBLE.getName + case BooleanType => CarbonType.DOUBLE.getName + case TimestampType => CarbonType.TIMESTAMP.getName case other => sys.error(s"unsupported type: $other") } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a82a1270/integration/spark/src/test/resources/test.json ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/test.json b/integration/spark/src/test/resources/test.json new file mode 100644 index 0000000..50a859c --- /dev/null +++ b/integration/spark/src/test/resources/test.json @@ -0,0 +1,3 @@ +{"name":"Michael"} +{"name":"Andy", "age":30} +{"name":"Justin", "age":19} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a82a1270/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala index 1da0533..2d5d3d5 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala @@ -173,6 +173,26 @@ class SparkDatasourceSuite extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE test") } + test("json data with long datatype issue CARBONDATA-405") { + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + val jsonDF = read.format("json").load("./src/test/resources/test.json") + jsonDF.write + .format("carbondata") + .option("tableName", "dftesttable") + .option("compress", "true") + .mode(SaveMode.Overwrite) + .save() + val carbonDF = read + .format("carbondata") + .option("tableName", "dftesttable") + .load() + checkAnswer( + carbonDF.select("age", "name"), + jsonDF.select("age", "name")) + sql("drop table dftesttable") + } + override def afterAll { sql("DROP TABLE IF EXISTS carbon1") sql("DROP TABLE IF EXISTS carbon2")
