[carbondata] branch branch-1.5 updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. raghunandan pushed a commit to branch branch-1.5 in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/branch-1.5 by this push: new 202365e [maven-release-plugin] prepare for next development iteration 202365e is described below commit 202365eb5c60c9b569d200df992829659a3ec2f6 Author: Raghunandan S AuthorDate: Mon Jan 21 19:53:59 2019 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- common/pom.xml| 2 +- core/pom.xml | 2 +- datamap/bloom/pom.xml | 2 +- datamap/examples/pom.xml | 2 +- datamap/lucene/pom.xml| 2 +- datamap/mv/core/pom.xml | 2 +- datamap/mv/plan/pom.xml | 2 +- examples/spark2/pom.xml | 2 +- format/pom.xml| 2 +- hadoop/pom.xml| 2 +- integration/hive/pom.xml | 2 +- integration/presto/pom.xml| 2 +- integration/spark-common-test/pom.xml | 2 +- integration/spark-common/pom.xml | 2 +- integration/spark-datasource/pom.xml | 2 +- integration/spark2/pom.xml| 2 +- pom.xml | 4 ++-- processing/pom.xml| 2 +- store/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 4b24730..b8af897 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index e0022ef..0f01c7b 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index e7ebe7d..ea93d30 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../pom.xml diff --git a/datamap/bloom/pom.xml b/datamap/bloom/pom.xml index 456911f..9070287 100644 --- a/datamap/bloom/pom.xml +++ b/datamap/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../pom.xml diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml index c427198..e88a906 100644 --- a/datamap/examples/pom.xml +++ b/datamap/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../pom.xml diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml index 03527db..6162f01 100644 --- a/datamap/lucene/pom.xml +++ b/datamap/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../pom.xml diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml index d432993..7924305 100644 --- a/datamap/mv/core/pom.xml +++ b/datamap/mv/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../../pom.xml diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml index f2efc97..1210331 100644 --- a/datamap/mv/plan/pom.xml +++ b/datamap/mv/plan/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../../pom.xml diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml index d723e54..d6804c9 100644 --- a/examples/spark2/pom.xml +++ b/examples/spark2/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index 7afd6ef..8284565 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index c9cdcc7..e23c74b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../pom.xml diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml index a45ab42..e2c1504 100644 --- a/integration/hive/pom.xml +++ b/integration/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2 +1.5.3-SNAPSHOT ../../pom.xml diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index 78a75fc..0ecd4a3 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@ -22,7 +22,7
[carbondata] annotated tag apache-carbondata-1.5.2-rc1 created (now 5e7254e)
This is an automated email from the ASF dual-hosted git repository. raghunandan pushed a change to annotated tag apache-carbondata-1.5.2-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 5e7254e (tag) tagging a8235fa3dd2d73497b6a9b7c57fd78fe589cd0cf (commit) replaces apache-carbondata-1.5.1-rc2 by Raghunandan S on Mon Jan 21 19:53:38 2019 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-1.5.2-rc1 --- No new revisions were added by this update.
[carbondata] branch branch-1.5 updated: [maven-release-plugin] prepare release apache-carbondata-1.5.2-rc1
This is an automated email from the ASF dual-hosted git repository. raghunandan pushed a commit to branch branch-1.5 in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/branch-1.5 by this push: new a8235fa [maven-release-plugin] prepare release apache-carbondata-1.5.2-rc1 a8235fa is described below commit a8235fa3dd2d73497b6a9b7c57fd78fe589cd0cf Author: Raghunandan S AuthorDate: Mon Jan 21 19:53:03 2019 +0530 [maven-release-plugin] prepare release apache-carbondata-1.5.2-rc1 --- assembly/pom.xml | 2 +- common/pom.xml| 2 +- core/pom.xml | 2 +- datamap/bloom/pom.xml | 2 +- datamap/examples/pom.xml | 2 +- datamap/lucene/pom.xml| 2 +- datamap/mv/core/pom.xml | 2 +- datamap/mv/plan/pom.xml | 2 +- examples/spark2/pom.xml | 2 +- format/pom.xml| 2 +- hadoop/pom.xml| 2 +- integration/hive/pom.xml | 2 +- integration/presto/pom.xml| 2 +- integration/spark-common-test/pom.xml | 2 +- integration/spark-common/pom.xml | 2 +- integration/spark-datasource/pom.xml | 2 +- integration/spark2/pom.xml| 2 +- pom.xml | 4 ++-- processing/pom.xml| 2 +- store/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index a3e1cfd..4b24730 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index e10b48b..e0022ef 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 6a22d94..e7ebe7d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../pom.xml diff --git a/datamap/bloom/pom.xml b/datamap/bloom/pom.xml index c245599..456911f 100644 --- a/datamap/bloom/pom.xml +++ b/datamap/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../pom.xml diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml index c1823c2..c427198 100644 --- a/datamap/examples/pom.xml +++ b/datamap/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../pom.xml diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml index 74999aa..03527db 100644 --- a/datamap/lucene/pom.xml +++ b/datamap/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../pom.xml diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml index 3209c91..d432993 100644 --- a/datamap/mv/core/pom.xml +++ b/datamap/mv/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../../pom.xml diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml index 7809337..f2efc97 100644 --- a/datamap/mv/plan/pom.xml +++ b/datamap/mv/plan/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../../pom.xml diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml index b47b0af..d723e54 100644 --- a/examples/spark2/pom.xml +++ b/examples/spark2/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index e923bfb..7afd6ef 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index fcba526..c9cdcc7 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../pom.xml diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml index 7b8b06e..a45ab42 100644 --- a/integration/hive/pom.xml +++ b/integration/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -1.5.2-SNAPSHOT +1.5.2 ../../pom.xml diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index fd955dd..78a75fc 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@
[carbondata] branch branch-1.5 updated (951a781 -> e14033c)
This is an automated email from the ASF dual-hosted git repository. ravipesala pushed a change to branch branch-1.5 in repository https://gitbox.apache.org/repos/asf/carbondata.git. from 951a781 [CARBONDATA-3175]Fix Testcase failures in complex delimiters new 27d4713 [CARBONDATA-3118] support parallel block pruning for non-default datamaps new 85fc4db [CARBONDATA-3073] Support configure TableProperties,withLoadOption etc. interface in carbon writer of C++ SDK support withTableProperty, withLoadOption,taskNo, uniqueIdentifier, withThreadSafe,withBlockSize, withBlockletSize, localDictionaryThreshold, enableLocalDictionary, sortBy in C++ SDK new 1a5aecf [CARBONDATA-3119] Fixed SDK Write for Complex Array Type when Array is Empty new c501669 [HOTFIX] Fixed Query performance issue new c6e2ad4 [CARBONDATA-3160] Compaction support with MAP data type new 5ebb657 [CARBONDATA-3174]varchar column trailing space issue fixed new 3ae9b53 [CARBONDATA-3162][CARBONDATA-3163][CARBONDATA-3164] 'no_sort' as default sort_scope new 3d62b87 [CARBONDATA-2999] support read schema from S3 new 3abb174 [CARBONDATA-3182] Fixed SDV Testcase failures new e3eaa75 [HOTFIX] replace apache common log with carbondata log4j new 941d35f [SDV] Add datasource testcases for Spark File Format new a423415 [CARBONDATA-3181][BloomDataMap] Fix access field error for BitSet in bloom filter new 04e4d91 [CARBONDATA-3187] Supported Global Dictionary For Map new a1be8f9 [CARBONDATA-3161]Pipe dilimiter is not working for streaming table new 27d66da [CARBONDATA-3149]Support alter table column rename new 574d469 [CARBONDATA-3179] Map Data Load Failure and Struct Projection Pushdown Issue new dd667ba [CARBONDATA-3127]Fix the HiveExample & TestCarbonSerde exception new afd8f4e [CARBONDATA-3186]Avoid creating empty carbondata file when all the records are bad record with action redirect. new 58c69d7 [CARBONDATA-3192] Fix for compaction compatibilty issue new 577528f [CARBONDATA-3184]Fix DataLoad Failure with 'using carbondata' new a672e43 [CARBONDATA-3080] Supporting local dictionary enable by default for SDK new 15717fa [CARBONDATA-3142]Add timestamp with thread name which created by CarbonThreadFactory new b52387e [CARBONDATA-3176] Optimize quick-start-guide documentation new 1c46415 [CARBONDATA-3188] Create carbon table as hive understandable metastore table needed by Presto and Hive new 2019c3d [CARBONDATA-3165]Protection of Bloom Null Exception new 44ec43a [HOTFIX] rename field thread_pool_size to match camel case new 679ef3b [CARBONDATA-2218] AlluxioCarbonFile while trying to force rename causes a FileSytem error and is not a DistributionFileSystem.(Adding Alluxio Support) new 7196118 [Carbondata-3173] Add the hive/presto documents index to the root of file ReadMe.md new cd5c33c [CARBONDATA-3195]Added validation for Inverted Index columns and added a test case in case of varchar new 0bcc84c [CARBONDATA-3196] [CARBONDATA-3203]Fixed Compaction for Complex types with Dictionary Include and also supported Compaction for restructured table new 9ec889f [CARBONDATA-3194] Integrating Carbon with Presto using hive connector new 6379f1c [CARBONDATA-3126] Correct some spell errors in CarbonData new b649f3c [CARBONDATA-3202]update the schema to session catalog after add column, drop column and column rename new 8530f45 [CARBONDATA-3208] Remove unused parameters, imports and optimize the spell errors new 19733b4 [CARBONDATA-3209] Remove unused import new 3ca5af8 [CARBONDATA-3213] Add license for java and md files new 6b723b0 [CARBONDATA-3206] Fix some spell errors in CarbonData new 35a82ef [CARBONDATA-3205]Fix Get Local Dictionary for empty Array of String new 24bf76e [CARBONDATA-3218] Fix schema refresh and wrong query result issues in presto. new e8c70af [CARBONDATA-3212] Fixed NegativeArraySizeException while querying in specific scenario new 74cb116 [HOTFIX] Optimize the code style in csdk/sdk markdown doc new 7bc5c13 [CARBONDATA-3216] Fix enableLocalDictionary with false issue in CSDK new c1116c4 [CARBONDATA-3217] Optimize implicit filter expression performance by removing extra serialization new 418084b [CARBONDATA-3149] Documentation for alter table column rename new 59eca1e [CARBONDATA-3226] Remove duplicated and useless files new 083228d [CARBONDATA-3223] Fixed Wrong Datasize and Indexsize calculation for old store using Show Segments new 05ad0c9 [CARBONDATA-3189] Fix PreAggregate Datamap Issue new cf62d0f [CARBONDATA-3222]Fix dataload failure after creation of preaggregate datamap on main table with long_string_columns new 48b0584 [CARBONDATA-3224] Support SDK validate the improper value when using withLoadOptions new cd25019
[carbondata] branch master updated: [CARBONDATA-3261] support float and byte data type reading from presto carbon
This is an automated email from the ASF dual-hosted git repository. ravipesala pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 556e220 [CARBONDATA-3261] support float and byte data type reading from presto carbon 556e220 is described below commit 556e220808946c009eb11bf36f9a2fd44cb49dfd Author: ajantha-bhat AuthorDate: Mon Jan 21 08:03:02 2019 +0530 [CARBONDATA-3261] support float and byte data type reading from presto carbon [CARBONDATA-3261] support float and byte data type reading from presto problem: support float and byte reading from presto cause: currently float and byte cannot be read in presto due to code issue. It was going as double data type. Hence array out of bound issue used to come as float/byte read from double stream reader. solution: Implement a new stream reader for float and byte. This closes #3085 --- .../carbondata/presto/CarbonVectorBatch.java | 6 ++ .../presto/PrestoCarbonVectorizedRecordReader.java | 14 +-- .../presto/readers/ByteStreamReader.java | 105 + .../presto/readers/FloatStreamReader.java | 105 + .../PrestoTestNonTransactionalTableFiles.scala | 63 +++-- 5 files changed, 273 insertions(+), 20 deletions(-) diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java index 2f0c9eb..aa8b062 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java @@ -28,8 +28,10 @@ import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.datatype.StructField; import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl; import org.apache.carbondata.presto.readers.BooleanStreamReader; +import org.apache.carbondata.presto.readers.ByteStreamReader; import org.apache.carbondata.presto.readers.DecimalSliceStreamReader; import org.apache.carbondata.presto.readers.DoubleStreamReader; +import org.apache.carbondata.presto.readers.FloatStreamReader; import org.apache.carbondata.presto.readers.IntegerStreamReader; import org.apache.carbondata.presto.readers.LongStreamReader; import org.apache.carbondata.presto.readers.ObjectStreamReader; @@ -89,6 +91,10 @@ public class CarbonVectorBatch { return new LongStreamReader(batchSize, field.getDataType(), dictionary); } else if (dataType == DataTypes.DOUBLE) { return new DoubleStreamReader(batchSize, field.getDataType(), dictionary); +} else if (dataType == DataTypes.FLOAT) { + return new FloatStreamReader(batchSize, field.getDataType(), dictionary); +} else if (dataType == DataTypes.BYTE) { + return new ByteStreamReader(batchSize, field.getDataType(), dictionary); } else if (dataType == DataTypes.STRING) { return new SliceStreamReader(batchSize, field.getDataType(), dictionary); } else if (DataTypes.isDecimal(dataType)) { diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java index 08401df..3b27557 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java @@ -26,7 +26,6 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; -import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.datatype.StructField; import org.apache.carbondata.core.metadata.encoder.Encoding; @@ -201,17 +200,8 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader { } for (ProjectionMeasure msr : queryMeasures) { - DataType dataType = msr.getMeasure().getDataType(); - if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT - || dataType == DataTypes.LONG) { -fields[msr.getOrdinal()] = -new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); - } else if (DataTypes.isDecimal(dataType)) { -fields[msr.getOrdinal()] = -new StructField(msr.getColumnName(),
[carbondata] branch master updated: [SDV] Fix SDV testcase failures
This is an automated email from the ASF dual-hosted git repository. ravipesala pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new e485e14 [SDV] Fix SDV testcase failures e485e14 is described below commit e485e14a8a3f680190aedaa540755a432ea4d7c3 Author: shivamasn AuthorDate: Thu Jan 17 18:12:23 2019 +0530 [SDV] Fix SDV testcase failures problem: Dictionary Include test cases were failing because the plan was incorrect. solution: Updated the plan in the QueryTest. This closes #3081 --- .../src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala index 3ee75ad..39beae1 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala @@ -44,8 +44,6 @@ class QueryTest extends PlanTest with Suite { // Add Locale setting Locale.setDefault(Locale.US) - CarbonProperties.getInstance() -.addProperty(CarbonCommonConstants.VALIDATE_DIRECT_QUERY_ON_DATAMAP, "false") /** * Runs the plan and makes sure the answer contains all of the keywords, or the @@ -139,7 +137,7 @@ class QueryTest extends PlanTest with Suite { val sqlContext: SQLContext = TestQueryExecutor.INSTANCE.sqlContext - sqlContext.sparkSession.experimental.extraOptimizations = Seq(new CarbonFileIndexReplaceRule) + sqlContext.sparkSession.experimental.extraOptimizations :+ Seq(new CarbonFileIndexReplaceRule) val resourcesPath = TestQueryExecutor.resourcesPath
[carbondata] branch master updated: [CARBONDATA-3242] Move Range_Column into the table level properties
This is an automated email from the ASF dual-hosted git repository. ravipesala pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 9f13d2b [CARBONDATA-3242] Move Range_Column into the table level properties 9f13d2b is described below commit 9f13d2b6c7639330a2228f834820131c714bb80b Author: QiangCai AuthorDate: Thu Jan 10 21:03:14 2019 +0800 [CARBONDATA-3242] Move Range_Column into the table level properties Move Range_Column into the table level properties This closes #3063 --- .../core/constants/CarbonCommonConstants.java | 11 .../core/constants/CarbonLoadOptionConstants.java | 6 -- .../core/metadata/schema/table/CarbonTable.java| 10 +++ .../carbondata/core/util/CarbonProperties.java | 24 +++ .../dataload/TestRangeColumnDataLoad.scala | 74 ++ .../spark/load/DataLoadProcessBuilderOnSpark.scala | 5 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala| 18 +- .../command/management/CarbonLoadDataCommand.scala | 37 +-- .../org/apache/spark/util/AlterTableUtil.scala | 25 +++- .../loading/model/CarbonLoadModelBuilder.java | 9 --- 10 files changed, 170 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index c1ef940..ccc8b99 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -426,6 +426,7 @@ public final class CarbonCommonConstants { */ public static final String DICTIONARY_PATH = "dictionary_path"; public static final String SORT_COLUMNS = "sort_columns"; + public static final String RANGE_COLUMN = "range_column"; public static final String PARTITION_TYPE = "partition_type"; public static final String NUM_PARTITIONS = "num_partitions"; public static final String RANGE_INFO = "range_info"; @@ -1176,6 +1177,16 @@ public final class CarbonCommonConstants { */ public static final int SORT_SIZE_MIN_VAL = 1000; + /** + * For Range_Column, it will use SCALE_FACTOR to control the size of each partition. + * When SCALE_FACTOR is the compression ratio of carbonData, + * each task will generate one CarbonData file. + * And the size of this CarbonData file is about TABLE_BLOCKSIZE of this table. + */ + public static final String CARBON_RANGE_COLUMN_SCALE_FACTOR = "carbon.range.column.scale.factor"; + + public static final String CARBON_RANGE_COLUMN_SCALE_FACTOR_DEFAULT = "3"; + // // Query parameter start here // diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java index eef2bef..225a8aa 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java @@ -172,10 +172,4 @@ public final class CarbonLoadOptionConstants { public static final String CARBON_LOAD_SORT_MEMORY_SPILL_PERCENTAGE_DEFAULT = "0"; - /** - * For Range_Column, it will use SCALE_FACTOR to control the size of each partition. - * When SCALE_FACTOR is about the compression ratio, each task will generate one CarbonData file. - * And the size of the file is about TABLE_BLOCKSIZE of this table. - */ - public static final int CARBON_RANGE_COLUMN_SCALE_FACTOR_DEFAULT = 3; } diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index daaed9d..f89dd6c 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -947,6 +947,16 @@ public class CarbonTable implements Serializable { return numberOfNoDictSortColumns; } + public CarbonColumn getRangeColumn() { +String rangeColumn = + tableInfo.getFactTable().getTableProperties().get(CarbonCommonConstants.RANGE_COLUMN); +if (rangeColumn == null) { + return null; +} else { + return getColumnByName(getTableName(), rangeColumn); +} + } + public TableInfo getTableInfo() { return tableInfo; } diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index