Repository: carbondata Updated Branches: refs/heads/master cd0ce4187 -> ceb135175
[CARBONDATA-3098] Fix for negative exponents value giving wrong results in Float datatype Problem: When the value of exponent is a negative number then the data is incorrect due to loss of precision of Floating point values and wrong calculation of the count of decimal points. Solution: Handled floating point precision by converting it to double and counted the decimal count values as done in double datatype(using Big Decimal). This closes #2918 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ceb13517 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ceb13517 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ceb13517 Branch: refs/heads/master Commit: ceb13517553e729ccb408d95ae7ae401f7aebcb8 Parents: cd0ce41 Author: Manish Nalla <manishnalla1...@gmail.com> Authored: Wed Nov 14 10:57:49 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Wed Nov 14 14:03:55 2018 +0530 ---------------------------------------------------------------------- .../encoding/adaptive/AdaptiveFloatingCodec.java | 14 +------------- .../page/statistics/PrimitivePageStatsCollector.java | 14 +------------- .../datasource/SparkCarbonDataSourceTest.scala | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java index 49696eb..b04c9df 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java @@ -167,19 +167,7 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec { @Override public void encode(int rowId, float value) { - if (targetDataType == DataTypes.BYTE) { - encodedPage.putByte(rowId, (byte) (value * floatFactor)); - } else if (targetDataType == DataTypes.SHORT) { - encodedPage.putShort(rowId, (short) (value * floatFactor)); - } else if (targetDataType == DataTypes.SHORT_INT) { - encodedPage.putShortInt(rowId, (int) (value * floatFactor)); - } else if (targetDataType == DataTypes.INT) { - encodedPage.putInt(rowId, (int) (value * floatFactor)); - } else if (targetDataType == DataTypes.LONG) { - encodedPage.putLong(rowId, (long) (value * floatFactor)); - } else { - throw new RuntimeException("internal error: " + debugInfo()); - } + encode(rowId, (double) value); } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java index 9be5a58..e604057 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java @@ -253,19 +253,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si } private int getDecimalCount(float value) { - int decimalPlaces = 0; - try { - String strValue = Float.valueOf(Math.abs(value)).toString(); - int integerPlaces = strValue.indexOf('.'); - if (-1 != integerPlaces) { - decimalPlaces = strValue.length() - integerPlaces - 1; - } - } catch (NumberFormatException e) { - if (!Double.isInfinite(value)) { - throw e; - } - } - return decimalPlaces; + return getDecimalCount((double) value); } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala index 1e58a9e..5eb103a 100644 --- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala +++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala @@ -1310,6 +1310,21 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll { checkAnswer(spark.sql("select * from t_carbn01b_hive"), spark.sql("select * from t_carbn01b")) spark.sql("drop table if exists t_carbn01b_hive") spark.sql(s"drop table if exists t_carbn01b") + } + + test("Test Float value by having negative exponents") { + spark.sql("DROP TABLE IF EXISTS float_p") + spark.sql("DROP TABLE IF EXISTS float_c") + spark.sql("CREATE TABLE float_p(f float) using parquet") + spark.sql("CREATE TABLE float_c(f float) using carbon") + spark.sql("INSERT INTO float_p select \"1.4E-3\"") + spark.sql("INSERT INTO float_p select \"1.4E-38\"") + spark.sql("INSERT INTO float_c select \"1.4E-3\"") + spark.sql("INSERT INTO float_c select \"1.4E-38\"") + checkAnswer(spark.sql("SELECT * FROM float_p"), + spark.sql("SELECT * FROM float_c")) + spark.sql("DROP TABLE float_p") + spark.sql("DROP TABLE float_c") } test("test fileformat flow with drop and query on same table") {