[CARBONDATA-2775] Adaptive encoding fails for Unsafe OnHeap. if, target datatype is SHORT_INT
problem: [CARBONDATA-2775] Adaptive encoding fails for Unsafe OnHeap if, target data type is SHORT_INT solution: If ENABLE_OFFHEAP_SORT = false, in carbon property. UnsafeFixLengthColumnPage.java will use different compress logic. Not the raw compression. In that case, for SHORT_INT data type , conversion need to handle. This closes #2546 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9767a586 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9767a586 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9767a586 Branch: refs/heads/branch-1.4 Commit: 9767a586f99c8ae20582ea29fa5dd58b362e0303 Parents: d720372 Author: ajantha-bhat <ajanthab...@gmail.com> Authored: Tue Jul 24 12:33:47 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Tue Jul 31 00:11:26 2018 +0530 ---------------------------------------------------------------------- .../page/UnsafeFixLengthColumnPage.java | 2 + ...UnsafeHeapColumnPageForComplexDataType.scala | 61 ++++++++++++++++++++ 2 files changed, 63 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/9767a586/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java index bcb74c0..f75deb6 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java @@ -495,6 +495,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { return totalLength / ByteUtil.SIZEOF_BYTE; } else if (dataType == DataTypes.SHORT) { return totalLength / ByteUtil.SIZEOF_SHORT; + } else if (dataType == DataTypes.SHORT_INT) { + return totalLength / ByteUtil.SIZEOF_SHORT_INT; } else if (dataType == DataTypes.INT) { return totalLength / ByteUtil.SIZEOF_INT; } else if (dataType == DataTypes.LONG) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/9767a586/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala new file mode 100644 index 0000000..acf75c1 --- /dev/null +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.integration.spark.testsuite.complexType + +import java.io.File + +import org.apache.spark.sql.test.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + +/** + * Test class of Adaptive Encoding UnSafe Column Page with Complex Data type + * + */ + +class TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType + extends QueryTest with BeforeAndAfterAll with TestAdaptiveComplexType { + + override def beforeAll(): Unit = { + + new File(CarbonProperties.getInstance().getSystemFolderLocation).delete() + sql("DROP TABLE IF EXISTS adaptive") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, + "true") + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, + "false") + } + + override def afterAll(): Unit = { + sql("DROP TABLE IF EXISTS adaptive") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, + "true") + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, + "true") + } + + +}
