[CARBONDATA-2216][Test] Fix bugs in sdv tests This closes #2012
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f9291cdb Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f9291cdb Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f9291cdb Branch: refs/heads/master Commit: f9291cdb3b480d3ad46f963b5c772b8fa3185034 Parents: 5397c05 Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Wed Feb 28 16:02:55 2018 +0800 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Mar 8 22:21:11 2018 +0530 ---------------------------------------------------------------------- .../core/datamap/dev/expr/AndDataMapExprWrapper.java | 2 +- .../core/datamap/dev/expr/DataMapExprWrapper.java | 2 +- .../datamap/dev/expr/DataMapExprWrapperImpl.java | 2 +- .../core/datamap/dev/expr/OrDataMapExprWrapper.java | 2 +- .../hadoop/api/CarbonTableInputFormat.java | 3 ++- .../cluster/sdv/generated/MergeIndexTestCase.scala | 15 +++++++++++++-- .../preaaggregate/PreAggregateTableHelper.scala | 5 ----- .../apache/spark/sql/optimizer/CarbonFilters.scala | 4 +--- 8 files changed, 20 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java index 12b60b4..74469d7 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.DataMapLevel; +import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java index ddb19e9..14cfc33 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.io.Serializable; import java.util.List; -import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.DataMapLevel; +import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java index d4be416..c6b011c 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java @@ -22,8 +22,8 @@ import java.util.List; import java.util.UUID; import org.apache.carbondata.core.datamap.DataMapDistributable; -import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.DataMapLevel; +import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.TableDataMap; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java index a8bfcad..37cd5dd 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java @@ -22,8 +22,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.DataMapLevel; +import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java index 81b5fde..bcc487e 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java @@ -929,7 +929,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> { TableDataMap blockletMap = DataMapStoreManager.getInstance().getDefaultDataMap(identifier); LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager .readTableStatusFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath())); - SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier, loadMetadataDetails); + SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager( + identifier, loadMetadataDetails); SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments = new SegmentStatusManager(identifier).getValidAndInvalidSegments(loadMetadataDetails); Map<String, Long> blockRowCountMapping = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala index c0abe4e..b886b11 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala @@ -92,18 +92,29 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll { val table = CarbonMetadata.getInstance().getCarbonTable("default","carbon_automation_nonmerge") new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false) assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 0) + assert(getMergedIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 1) checkAnswer(sql("""Select count(*) from carbon_automation_nonmerge"""), rows) } private def getIndexFileCount(dbName: String, tableName: String, segment: String): Int = { + getFileCount(dbName, tableName, segment, CarbonTablePath.INDEX_FILE_EXT) + } + + private def getMergedIndexFileCount(dbName: String, tableName: String, segment: String): Int = { + getFileCount(dbName, tableName, segment, CarbonTablePath.MERGE_INDEX_FILE_EXT) + } + + private def getFileCount(dbName: String, + tableName: String, + segment: String, + suffix: String): Int = { val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName, tableName) val identifier = carbonTable.getAbsoluteTableIdentifier val path = CarbonTablePath .getSegmentPath(identifier.getTablePath, segment) val carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter { override def accept(file: CarbonFile): Boolean = { - file.getName.endsWith(CarbonTablePath - .INDEX_FILE_EXT) + file.getName.endsWith(suffix) } }) if (carbonFiles != null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala index 4c0e637..1f1e1e6 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala @@ -26,16 +26,11 @@ import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil -import org.apache.spark.sql.optimizer.CarbonFilters import org.apache.spark.sql.parser.CarbonSpark2SqlParser import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider -import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema import org.apache.carbondata.core.metadata.schema.table.CarbonTable -import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider -import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema} import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager} /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala index 2213fdb..3344449 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala @@ -40,11 +40,9 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression} import org.apache.carbondata.core.scan.expression.conditional._ import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression} -import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo} import org.apache.carbondata.core.scan.filter.intf.ExpressionType -import org.apache.carbondata.core.util.ThreadLocalSessionInfo -import org.apache.carbondata.core.util.path.CarbonTablePath import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.core.util.ThreadLocalSessionInfo import org.apache.carbondata.datamap.TextMatch import org.apache.carbondata.spark.CarbonAliasDecoderRelation import org.apache.carbondata.spark.util.CarbonScalaUtil
