This is an automated email from the ASF dual-hosted git repository. qiangcai pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git.
from 8ff487f [CARBONDATA-3548] Polygon expression processing using unknown expression and filtering performance improvement new 42b15ec add Secondary index feature for better pruning new f127245 [CARBONDATA-3680] core changes and load changes for SI integration The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/constants/CarbonCommonConstants.java | 74 + .../core/datamap/AbstractDataMapJob.java | 8 + .../carbondata/core/datamap/DataMapUtil.java | 46 + .../carbondata/core/datamap/TableDataMap.java | 27 + .../core/datamap/dev/CacheableDataMap.java | 10 + .../core/datamap/dev/DataMapFactory.java | 7 + .../datamap/dev/expr/AndDataMapExprWrapper.java | 2 +- .../core/datamap/dev/expr/DataMapExprWrapper.java | 36 +- .../datamap/dev/expr/DataMapExprWrapperImpl.java | 10 +- .../datamap/dev/expr/OrDataMapExprWrapper.java | 2 +- .../core/datastore/impl/DFSFileReaderImpl.java | 4 +- .../core/indexstore/AbstractMemoryDMStore.java | 12 + .../core/indexstore/BlockletDataMapIndexStore.java | 17 +- .../carbondata/core/indexstore/SegmentWrapper.java | 65 + .../core/indexstore/SegmentWrapperContainer.java | 64 + .../TableBlockIndexUniqueIdentifier.java | 4 + .../TableBlockIndexUniqueIdentifierWrapper.java | 24 + .../core/indexstore/UnsafeMemoryDMStore.java | 22 + .../indexstore/blockletindex/BlockDataMap.java | 27 + .../blockletindex/BlockletDataMapFactory.java | 58 + .../blockletindex/BlockletDataMapModel.java | 9 +- .../carbondata/core/metadata/SegmentFileStore.java | 2 +- .../metadata/schema/indextable/IndexMetadata.java | 128 + .../metadata/schema/indextable/IndexTableInfo.java | 147 + .../core/metadata/schema/table/CarbonTable.java | 62 + .../carbondata/core/mutate/CarbonUpdateUtil.java | 49 + .../core/mutate/SegmentUpdateDetails.java | 7 + .../core/scan/wrappers/ByteArrayWrapper.java | 2 +- .../core/statusmanager/SegmentStatusManager.java | 20 + .../carbondata/core/util/BlockletDataMapUtil.java | 29 + .../carbondata/core/util/CarbonProperties.java | 197 +- .../apache/carbondata/core/util/CarbonUtil.java | 6 +- .../apache/carbondata/core/util/SessionParams.java | 2 + .../carbondata/core/util/path/CarbonTablePath.java | 7 +- .../apache/carbondata/hadoop/CarbonInputSplit.java | 1 - dev/findbugs-exclude.xml | 6 + .../carbondata/hadoop/api/CarbonInputFormat.java | 30 +- .../hadoop/api/CarbonTableInputFormat.java | 76 +- .../hadoop/api/CarbonTableOutputFormat.java | 4 +- .../src/test/resources/secindex/IUD/sample_1.csv | 123 + .../src/test/resources/secindex/IUD/sample_2.csv | 30 + .../src/test/resources/secindex/data_10000.csv | 10390 +++++++++++++++++++ .../src/test/resources/secindex/datafile_100.csv | 10 + .../resources/{IUD/comp1.csv => secindex/dest.csv} | 0 .../{IUD/comp2.csv => secindex/dest1.csv} | 0 .../{IUD/comp3.csv => secindex/dest2.csv} | 0 .../{IUD/comp4.csv => secindex/dest3.csv} | 0 .../src/test/resources/secindex/firstunique.csv | 1 + .../src/test/resources/secindex/index.csv | 1 + .../resources/secindex/secondaryIndexLikeTest.csv | 10 + .../src/test/resources/secindex/secondunique.csv | 1 + .../test/resources/{IUD => secindex}/source3.csv | 0 .../StandardPartitionTableLoadingTestCase.scala | 3 +- .../events/CreateCarbonRelationEvent.scala | 28 +- .../org/apache/carbondata/events/Events.scala | 8 + .../carbondata/spark/rdd/CarbonScanRDD.scala | 20 +- .../carbondata/streaming/StreamSinkFactory.scala | 10 +- .../org/apache/spark/rdd/CarbonMergeFilesRDD.scala | 2 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 + .../streaming/CarbonAppendableStreamSink.scala | 9 +- .../apache/spark/sql/test/TestQueryExecutor.scala | 5 +- .../datamap/IndexDataMapRebuildRDD.scala | 11 +- .../carbondata/indexserver/DataMapJobs.scala | 5 + .../indexserver/DistributedPruneRDD.scala | 3 + .../carbondata/indexserver/IndexServer.scala | 19 +- .../carbondata/indexserver/SegmentPruneRDD.scala | 65 + .../apache/spark/sql/CarbonCatalystOperators.scala | 1 + .../scala/org/apache/spark/sql/CarbonEnv.scala | 34 +- .../scala/org/apache/spark/sql/CarbonSource.scala | 3 +- .../scala/org/apache/spark/sql/CarbonUtils.scala | 8 + .../datamap/CarbonCreateDataMapCommand.scala | 5 + .../command/management/CarbonLoadDataCommand.scala | 11 +- .../mutation/CarbonProjectForUpdateCommand.scala | 1 + .../mutation/merge/CarbonMergeDataSetCommand.scala | 11 +- .../strategy/CarbonLateDecodeStrategy.scala | 130 +- .../spark/sql/execution/strategy/DDLStrategy.scala | 47 + .../spark/sql/hive/CarbonFileMetastore.scala | 21 +- .../spark/sql/hive/CarbonHiveMetadataUtil.scala | 108 +- .../org/apache/spark/sql/hive/CarbonRelation.scala | 23 +- .../CreateCarbonSourceTableAsSelectCommand.scala | 1 - .../execution/command/CarbonHiveCommands.scala | 18 +- .../apache/spark/sql/optimizer/CarbonFilters.scala | 4 +- .../spark/sql/parser/CarbonSpark2SqlParser.scala | 145 +- .../Jobs/BlockletDataMapDetailsWithSchema.java | 67 + .../Jobs/CarbonBlockLoaderHelper.java | 98 + .../Jobs/DistributableBlockletDataMapLoader.java | 194 + .../Jobs/SparkBlockletDataMapLoaderJob.scala | 202 + .../secondaryindex/command/DropIndexCommand.scala | 188 + .../command/RegisterIndexTableCommand.scala | 103 + .../secondaryindex/command/SICreationCommand.scala | 574 + .../sql/secondaryindex/command/SILoadCommand.scala | 149 + .../command/SIRebuildSegmentCommand.scala | 202 + .../command/ShowIndexesCommand.scala | 164 + .../AlterTableColumnRenameEventListener.scala | 157 + .../AlterTableCompactionPostEventListener.scala | 125 + .../events/AlterTableDropColumnEventListener.scala | 91 + .../AlterTableMergeIndexSIEventListener.scala | 120 + .../events/AlterTableRenameEventListener.scala | 64 + .../events/CleanFilesPostEventListener.scala | 60 + .../events/CreateCarbonRelationEventListener.scala | 49 + .../events/DeleteFromTableEventListener.scala | 78 + .../events/DeleteSegmentByDateListener.scala | 58 + .../events/DeleteSegmentByIdListener.scala | 57 + .../events/DropCacheSIEventListener.scala | 61 + .../sql/secondaryindex/events/LoadSIEvents.scala | 50 + .../events/SIDropEventListener.scala | 126 + .../events/SILoadEventListener.scala | 90 + .../SILoadEventListenerForFailedSegments.scala | 198 + .../events/SIRefreshEventListener.scala | 45 + .../events/ShowCacheSIEventListener.scala | 65 + .../events/UpdateTablePreEventListener.scala | 56 + .../exception/IndexTableExistException.java | 36 +- .../exception/SecondaryIndexException.java | 27 +- .../hive/CarbonInternalMetastore.scala | 267 + .../joins/BroadCastSIFilterPushJoin.scala | 541 + .../load/CarbonInternalLoaderUtil.java | 328 + .../spark/sql/secondaryindex/load/Compactor.scala | 125 + .../load/RowComparatorWithOutKettle.java | 86 + .../optimizer/CarbonCostBasedOptimizer.java | 92 + .../optimizer/CarbonSITransformationRule.scala | 131 + .../optimizer/CarbonSecondaryIndexOptimizer.scala | 840 ++ .../query/CarbonSecondaryIndexExecutor.java | 154 + .../query/SecondaryIndexQueryResultProcessor.java | 496 + .../secondaryindex/rdd/CarbonSIRebuildRDD.scala | 352 + .../rdd/CarbonSecondaryIndexRDD.scala | 320 + .../secondaryindex/rdd/SecondaryIndexCreator.scala | 401 + .../util/CarbonInternalScalaUtil.scala | 383 + .../sql/secondaryindex/util/FileInternalUtil.scala | 129 + .../sql/secondaryindex/util/IndexTableUtil.java | 59 + .../sql/secondaryindex/util/InternalKeyVal.scala | 20 +- .../secondaryindex/util/SecondaryIndexUtil.scala | 565 + .../org/apache/spark/util/AlterTableUtil.scala | 2 +- .../apache/spark/sql/CarbonToSparkAdapter.scala | 6 +- .../apache/spark/sql/CarbonToSparkAdapter.scala | 6 +- pom.xml | 5 + processing/pom.xml | 5 + .../processing/loading/events/LoadEvents.java | 47 + .../loading/sort/unsafe/UnsafeCarbonRowPage.java | 9 +- .../loading/sort/unsafe/UnsafeSortDataRows.java | 33 +- secondary_index/pom.xml | 244 + .../CarbonDataFileMergeTestCaseOnSI.scala | 276 + .../CarbonIndexFileMergeTestCaseWithSI.scala | 289 + .../testsuite/secondaryindex/DropTableTest.scala | 77 + .../InsertIntoCarbonTableTestCase.scala | 47 + ...tAlterTableColumnRenameWithSecondaryIndex.scala | 113 + .../TestBroadCastSIFilterPushJoinWithUDF.scala | 415 + .../TestCTASWithSecondaryIndex.scala | 332 + .../secondaryindex/TestCacheOperationsForSI.scala | 124 + .../testsuite/secondaryindex/TestCarbonJoin.scala | 73 + .../TestCreateIndexForCleanAndDeleteSegment.scala | 95 + .../secondaryindex/TestCreateIndexTable.scala | 489 + .../TestCreateIndexWithLoadAndCompaction.scala | 267 + .../TestLikeQueryWithSecondaryIndex.scala | 95 + .../TestNIQueryWithSecondaryIndex.scala | 253 + .../TestRegisterIndexCarbonTable.scala | 86 + .../secondaryindex/TestSIWithAddSegment.scala | 113 + .../secondaryindex/TestSIWithSecondryIndex.scala | 250 + .../TestSecondaryIndexForORFilterPushDown.scala | 254 + .../TestSecondaryIndexWithAggQueries.scala | 178 + .../secondaryindex/TestSecondaryIndexWithIUD.scala | 465 + ...IndexWithIndexOnFirstColumnAndSortColumns.scala | 48 + .../TestSecondaryIndexWithLocalDictionary.scala | 94 + .../TestSecondaryIndexWithUnsafeColumnPage.scala | 46 + .../apache/spark/util/TestCarbonSegmentUtil.scala | 313 + 164 files changed, 26375 insertions(+), 248 deletions(-) create mode 100644 core/src/main/java/org/apache/carbondata/core/indexstore/SegmentWrapper.java create mode 100644 core/src/main/java/org/apache/carbondata/core/indexstore/SegmentWrapperContainer.java create mode 100644 core/src/main/java/org/apache/carbondata/core/metadata/schema/indextable/IndexMetadata.java create mode 100644 core/src/main/java/org/apache/carbondata/core/metadata/schema/indextable/IndexTableInfo.java create mode 100644 integration/spark-common-test/src/test/resources/secindex/IUD/sample_1.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/IUD/sample_2.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/data_10000.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/datafile_100.csv copy integration/spark-common-test/src/test/resources/{IUD/comp1.csv => secindex/dest.csv} (100%) copy integration/spark-common-test/src/test/resources/{IUD/comp2.csv => secindex/dest1.csv} (100%) copy integration/spark-common-test/src/test/resources/{IUD/comp3.csv => secindex/dest2.csv} (100%) copy integration/spark-common-test/src/test/resources/{IUD/comp4.csv => secindex/dest3.csv} (100%) create mode 100644 integration/spark-common-test/src/test/resources/secindex/firstunique.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/index.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/secondaryIndexLikeTest.csv create mode 100644 integration/spark-common-test/src/test/resources/secindex/secondunique.csv copy integration/spark-common-test/src/test/resources/{IUD => secindex}/source3.csv (100%) copy core/src/main/java/org/apache/carbondata/core/datamap/AbstractDataMapJob.java => integration/spark-common/src/main/scala/org/apache/carbondata/events/CreateCarbonRelationEvent.scala (63%) create mode 100644 integration/spark2/src/main/scala/org/apache/carbondata/indexserver/SegmentPruneRDD.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/BlockletDataMapDetailsWithSchema.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/CarbonBlockLoaderHelper.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/DistributableBlockletDataMapLoader.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/DropIndexCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/RegisterIndexTableCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/SICreationCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/SILoadCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/SIRebuildSegmentCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/command/ShowIndexesCommand.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/AlterTableColumnRenameEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/AlterTableCompactionPostEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/AlterTableDropColumnEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/AlterTableMergeIndexSIEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/AlterTableRenameEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/CleanFilesPostEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/CreateCarbonRelationEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/DeleteFromTableEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/DeleteSegmentByDateListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/DeleteSegmentByIdListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/DropCacheSIEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/LoadSIEvents.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/SIDropEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/SILoadEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/SILoadEventListenerForFailedSegments.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/SIRefreshEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/ShowCacheSIEventListener.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/events/UpdateTablePreEventListener.scala copy core/src/main/java/org/apache/carbondata/core/datamap/AbstractDataMapJob.java => integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/exception/IndexTableExistException.java (60%) copy core/src/main/java/org/apache/carbondata/core/datamap/AbstractDataMapJob.java => integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/exception/SecondaryIndexException.java (63%) create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/hive/CarbonInternalMetastore.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/load/CarbonInternalLoaderUtil.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/load/Compactor.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/load/RowComparatorWithOutKettle.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonCostBasedOptimizer.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSITransformationRule.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/query/CarbonSecondaryIndexExecutor.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/query/SecondaryIndexQueryResultProcessor.java create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/rdd/CarbonSIRebuildRDD.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/rdd/CarbonSecondaryIndexRDD.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/rdd/SecondaryIndexCreator.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/util/CarbonInternalScalaUtil.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/util/FileInternalUtil.scala create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/util/IndexTableUtil.java copy core/src/main/java/org/apache/carbondata/core/datamap/AbstractDataMapJob.java => integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/util/InternalKeyVal.scala (62%) create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/secondaryindex/util/SecondaryIndexUtil.scala create mode 100644 secondary_index/pom.xml create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/mergedata/CarbonDataFileMergeTestCaseOnSI.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/mergeindex/CarbonIndexFileMergeTestCaseWithSI.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/DropTableTest.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/InsertIntoCarbonTableTestCase.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestAlterTableColumnRenameWithSecondaryIndex.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestBroadCastSIFilterPushJoinWithUDF.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCTASWithSecondaryIndex.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCacheOperationsForSI.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCarbonJoin.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCreateIndexForCleanAndDeleteSegment.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCreateIndexTable.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestCreateIndexWithLoadAndCompaction.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestLikeQueryWithSecondaryIndex.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestNIQueryWithSecondaryIndex.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestRegisterIndexCarbonTable.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithAddSegment.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithSecondryIndex.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexForORFilterPushDown.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexWithAggQueries.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexWithIUD.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexWithIndexOnFirstColumnAndSortColumns.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexWithLocalDictionary.scala create mode 100644 secondary_index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSecondaryIndexWithUnsafeColumnPage.scala create mode 100644 secondary_index/src/test/scala/org/apache/spark/util/TestCarbonSegmentUtil.scala