[CARBONDATA-463] Extract code to spark-common. This closes #365
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d94b99f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d94b99f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d94b99f3 Branch: refs/heads/master Commit: d94b99f366465b2426bbcd8277f80651f2408770 Parents: 567fa51 66ccd30 Author: ravipesala <ravi.pes...@gmail.com> Authored: Wed Nov 30 13:21:06 2016 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Wed Nov 30 13:21:06 2016 +0530 ---------------------------------------------------------------------- .../examples/GenerateDictionaryExample.scala | 2 +- integration/spark-common/pom.xml | 2 +- .../MalformedCarbonCommandException.java | 83 ++ .../carbondata/spark/load/CarbonLoaderUtil.java | 973 ++++++++++++++++++ .../spark/load/DeleteLoadFolders.java | 259 +++++ .../spark/load/DeletedLoadMetadata.java | 53 + .../spark/merger/CarbonCompactionExecutor.java | 233 +++++ .../spark/merger/CarbonCompactionUtil.java | 283 ++++++ .../spark/merger/CarbonDataMergerUtil.java | 695 +++++++++++++ .../spark/merger/CompactionCallable.java | 44 + .../carbondata/spark/merger/CompactionType.java | 28 + .../spark/merger/NodeBlockRelation.java | 60 ++ .../spark/merger/NodeMultiBlockRelation.java | 59 ++ .../spark/merger/RowResultMerger.java | 336 +++++++ .../carbondata/spark/merger/TableMeta.java | 42 + .../spark/merger/TupleConversionAdapter.java | 85 ++ .../spark/partition/api/DataPartitioner.java | 54 + .../spark/partition/api/Partition.java | 42 + .../api/impl/DataPartitionerProperties.java | 87 ++ .../partition/api/impl/DefaultLoadBalancer.java | 69 ++ .../spark/partition/api/impl/PartitionImpl.java | 54 + .../api/impl/PartitionMultiFileImpl.java | 51 + .../api/impl/QueryPartitionHelper.java | 77 ++ .../api/impl/SampleDataPartitionerImpl.java | 151 +++ .../readsupport/SparkRowReadSupportImpl.java | 69 ++ .../carbondata/spark/splits/TableSplit.java | 129 +++ .../carbondata/spark/util/CarbonQueryUtil.java | 142 +++ .../carbondata/spark/util/LoadMetadataUtil.java | 61 ++ .../spark/CarbonAliasDecoderRelation.scala | 43 + .../spark/CarbonColumnValidator.scala | 36 + .../apache/carbondata/spark/CarbonFilters.scala | 391 ++++++++ .../apache/carbondata/spark/CarbonOption.scala | 48 + .../carbondata/spark/CarbonSparkFactory.scala | 59 ++ .../spark/DictionaryDetailHelper.scala | 63 ++ .../org/apache/carbondata/spark/KeyVal.scala | 89 ++ .../carbondata/spark/csv/CarbonCsvReader.scala | 182 ++++ .../spark/csv/CarbonCsvRelation.scala | 249 +++++ .../carbondata/spark/csv/CarbonTextFile.scala | 91 ++ .../carbondata/spark/csv/DefaultSource.scala | 183 ++++ .../spark/rdd/CarbonCleanFilesRDD.scala | 82 ++ .../spark/rdd/CarbonDataLoadRDD.scala | 598 ++++++++++++ .../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 91 ++ .../spark/rdd/CarbonDeleteLoadRDD.scala | 84 ++ .../spark/rdd/CarbonDropTableRDD.scala | 71 ++ .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 557 +++++++++++ .../carbondata/spark/rdd/CarbonMergerRDD.scala | 342 +++++++ .../spark/rdd/CarbonSparkPartition.scala | 35 + .../apache/carbondata/spark/rdd/Compactor.scala | 130 +++ .../spark/rdd/DataLoadCoalescedRDD.scala | 68 ++ .../spark/rdd/DataLoadPartitionCoalescer.scala | 363 +++++++ .../spark/tasks/DictionaryWriterTask.scala | 106 ++ .../spark/tasks/SortIndexWriterTask.scala | 59 ++ .../carbondata/spark/util/CarbonScalaUtil.scala | 195 ++++ .../carbondata/spark/util/CommonUtil.scala | 259 +++++ .../spark/util/DataTypeConverterUtil.scala | 74 ++ .../spark/util/GlobalDictionaryUtil.scala | 843 ++++++++++++++++ .../CarbonTableIdentifierImplicit.scala | 42 + .../execution/command/carbonTableSchema.scala | 359 +++++++ .../spark/sql/hive/DistributionUtil.scala | 167 ++++ .../CarbonDecoderOptimizerHelper.scala | 149 +++ .../scala/org/apache/spark/util/FileUtils.scala | 94 ++ .../apache/spark/util/ScalaCompilerUtil.scala | 35 + .../scala/org/apache/spark/util/SparkUtil.scala | 73 ++ .../spark/merger/CarbonCompactionExecutor.java | 233 ----- .../spark/merger/CarbonCompactionUtil.java | 284 ------ .../spark/merger/CompactionCallable.java | 44 - .../spark/merger/CompactionType.java | 28 - .../spark/merger/RowResultMerger.java | 336 ------- .../spark/merger/TupleConversionAdapter.java | 85 -- .../MalformedCarbonCommandException.java | 83 -- .../carbondata/spark/load/CarbonLoaderUtil.java | 976 ------------------- .../spark/load/DeleteLoadFolders.java | 259 ----- .../spark/load/DeleteLoadFromMetadata.java | 44 - .../spark/load/DeletedLoadMetadata.java | 53 - .../spark/merger/CarbonDataMergerUtil.java | 696 ------------- .../spark/merger/NodeBlockRelation.java | 60 -- .../spark/merger/NodeMultiBlockRelation.java | 59 -- .../spark/partition/api/DataPartitioner.java | 54 - .../spark/partition/api/Partition.java | 42 - .../api/impl/DataPartitionerProperties.java | 87 -- .../partition/api/impl/DefaultLoadBalancer.java | 69 -- .../spark/partition/api/impl/PartitionImpl.java | 54 - .../api/impl/PartitionMultiFileImpl.java | 51 - .../api/impl/QueryPartitionHelper.java | 77 -- .../api/impl/SampleDataPartitionerImpl.java | 151 --- .../readsupport/SparkRowReadSupportImpl.java | 69 -- .../carbondata/spark/splits/TableSplit.java | 129 --- .../carbondata/spark/util/CarbonQueryUtil.java | 142 --- .../carbondata/spark/util/LoadMetadataUtil.java | 61 -- .../spark/CarbonColumnValidator.scala | 36 - .../spark/CarbonDataFrameWriter.scala | 2 +- .../apache/carbondata/spark/CarbonFilters.scala | 391 -------- .../apache/carbondata/spark/CarbonOption.scala | 46 - .../carbondata/spark/CarbonSparkFactory.scala | 60 -- .../spark/DictionaryDetailHelper.scala | 62 -- .../org/apache/carbondata/spark/KeyVal.scala | 89 -- .../carbondata/spark/csv/CarbonCsvReader.scala | 182 ---- .../spark/csv/CarbonCsvRelation.scala | 248 ----- .../carbondata/spark/csv/CarbonTextFile.scala | 63 -- .../carbondata/spark/csv/DefaultSource.scala | 182 ---- .../spark/rdd/CarbonCleanFilesRDD.scala | 83 -- .../spark/rdd/CarbonDataLoadRDD.scala | 604 ------------ .../spark/rdd/CarbonDataRDDFactory.scala | 80 +- .../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 92 -- .../spark/rdd/CarbonDeleteLoadRDD.scala | 84 -- .../spark/rdd/CarbonDropTableRDD.scala | 72 -- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 558 ----------- .../carbondata/spark/rdd/CarbonMergerRDD.scala | 344 ------- .../carbondata/spark/rdd/CarbonScanRDD.scala | 8 +- .../apache/carbondata/spark/rdd/Compactor.scala | 133 --- .../spark/rdd/NewCarbonDataLoadRDD.scala | 1 - .../spark/tasks/DictionaryWriterTask.scala | 106 -- .../spark/tasks/SortIndexWriterTask.scala | 59 -- .../carbondata/spark/util/CarbonScalaUtil.scala | 219 ----- .../carbondata/spark/util/CommonUtil.scala | 251 ----- .../spark/util/DataTypeConverterUtil.scala | 55 -- .../spark/util/GlobalDictionaryUtil.scala | 875 ----------------- .../apache/spark/rdd/DataLoadCoalescedRDD.scala | 68 -- .../spark/rdd/DataLoadPartitionCoalescer.scala | 363 ------- .../spark/sql/CarbonCatalystOperators.scala | 8 +- .../org/apache/spark/sql/CarbonContext.scala | 4 +- .../sql/CarbonDatasourceHadoopRelation.scala | 15 +- .../spark/sql/CarbonDatasourceRelation.scala | 8 +- .../spark/sql/CarbonDictionaryDecoder.scala | 13 +- .../scala/org/apache/spark/sql/CarbonEnv.scala | 50 +- .../scala/org/apache/spark/sql/CarbonScan.scala | 35 +- .../org/apache/spark/sql/CarbonSparkUtil.scala | 45 + .../org/apache/spark/sql/CarbonSqlParser.scala | 8 +- .../CarbonTableIdentifierImplicit.scala | 42 - .../execution/command/carbonTableSchema.scala | 502 ++-------- .../apache/spark/sql/hive/CarbonMetastore.scala | 562 +++++++++++ .../spark/sql/hive/CarbonMetastoreCatalog.scala | 576 ----------- .../spark/sql/hive/CarbonStrategies.scala | 19 +- .../spark/sql/hive/DistributionUtil.scala | 145 --- .../execution/command/CarbonHiveCommands.scala | 8 +- .../CarbonDecoderOptimizerHelper.scala | 149 --- .../spark/sql/optimizer/CarbonOptimizer.scala | 24 +- .../scala/org/apache/spark/util/FileUtils.scala | 94 -- .../apache/spark/util/ScalaCompilerUtil.scala | 35 - .../org/apache/spark/util/SplitUtils.scala | 67 -- .../TestDataLoadPartitionCoalescer.scala | 2 - .../spark/util/AllDictionaryTestCase.scala | 2 +- .../AutoHighCardinalityIdentifyTestCase.scala | 2 +- .../util/ExternalColumnDictionaryTestCase.scala | 2 +- ...GlobalDictionaryUtilConcurrentTestCase.scala | 2 +- .../util/GlobalDictionaryUtilTestCase.scala | 2 +- .../scala/org/apache/spark/sql/CarbonScan.scala | 127 +++ .../execution/CarbonLateDecodeStrategy.scala | 130 +++ 148 files changed, 11561 insertions(+), 11292 deletions(-) ----------------------------------------------------------------------
