[CARBONDATA-2189] Add DataMapProvider developer interface Add developer interface for 2 types of DataMap:
1.IndexDataMap: DataMap that leveraging index to accelerate filter query 2.MVDataMap: DataMap that leveraging Materialized View to accelerate olap style query, like SPJG query (select, predicate, join, groupby) This PR adds support for following logic when creating and dropping the DataMap This closes #1987 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2117c077 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2117c077 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2117c077 Branch: refs/heads/datamap Commit: 2117c077b02c90941a49d07fc487195c1866df6b Parents: 5733413 Author: Jacky Li <jacky.li...@qq.com> Authored: Thu Feb 22 20:59:59 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Sun Feb 25 20:53:40 2018 +0800 ---------------------------------------------------------------------- .../sql/MalformedDataMapCommandException.java | 4 + .../core/constants/CarbonCommonConstants.java | 1 - .../carbondata/core/datamap/DataMapChooser.java | 5 +- .../core/datamap/DataMapRegistry.java | 37 + .../core/datamap/DataMapStoreManager.java | 83 +- .../carbondata/core/datamap/TableDataMap.java | 46 +- .../core/datamap/dev/AbstractDataMapWriter.java | 2 +- .../carbondata/core/datamap/dev/DataMap.java | 61 -- .../core/datamap/dev/DataMapFactory.java | 85 -- .../core/datamap/dev/IndexDataMap.java | 61 ++ .../core/datamap/dev/IndexDataMapFactory.java | 85 ++ .../cgdatamap/AbstractCoarseGrainDataMap.java | 24 - .../AbstractCoarseGrainDataMapFactory.java | 34 - .../AbstractCoarseGrainIndexDataMap.java | 24 + .../AbstractCoarseGrainIndexDataMapFactory.java | 34 + .../dev/expr/DataMapExprWrapperImpl.java | 2 +- .../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 - .../AbstractFineGrainDataMapFactory.java | 38 - .../AbstractFineGrainIndexDataMap.java | 24 + .../AbstractFineGrainIndexDataMapFactory.java | 38 + .../indexstore/BlockletDataMapIndexStore.java | 33 +- .../blockletindex/BlockletDataMap.java | 981 ------------------- .../blockletindex/BlockletDataMapFactory.java | 256 ----- .../blockletindex/BlockletDataMapModel.java | 2 +- .../blockletindex/BlockletIndexDataMap.java | 981 +++++++++++++++++++ .../BlockletIndexDataMapFactory.java | 256 +++++ .../core/metadata/schema/table/CarbonTable.java | 10 +- .../metadata/schema/table/DataMapSchema.java | 1 + .../schema/table/DataMapSchemaFactory.java | 17 +- .../core/metadata/schema/table/TableSchema.java | 3 +- .../blockletindex/TestBlockletDataMap.java | 66 -- .../blockletindex/TestBlockletIndexDataMap.java | 59 ++ .../datamap/examples/MinMaxDataMap.java | 151 --- .../datamap/examples/MinMaxDataMapFactory.java | 116 --- .../datamap/examples/MinMaxIndexDataMap.java | 148 +++ .../examples/MinMaxIndexDataMapFactory.java | 116 +++ .../MinMaxDataMapExample.scala | 4 +- docs/datamap-developer-guide.md | 16 + .../hadoop/api/CarbonTableInputFormat.java | 4 +- .../preaggregate/TestPreAggCreateCommand.scala | 4 +- .../TestPreAggregateTableSelection.scala | 4 +- .../timeseries/TestTimeSeriesCreateTable.scala | 4 +- .../testsuite/datamap/CGDataMapTestCase.scala | 379 ------- .../datamap/CGIndexDataMapTestCase.scala | 379 +++++++ .../testsuite/datamap/DataMapWriterSuite.scala | 216 ---- .../testsuite/datamap/FGDataMapTestCase.scala | 476 --------- .../datamap/FGIndexDataMapTestCase.scala | 474 +++++++++ .../datamap/IndexDataMapWriterSuite.scala | 216 ++++ .../testsuite/datamap/TestDataMapCommand.scala | 282 ------ .../datamap/TestIndexDataMapCommand.scala | 279 ++++++ .../iud/InsertOverwriteConcurrentTest.scala | 19 +- .../carbondata/datamap/DataMapManager.java | 53 + .../carbondata/datamap/DataMapProperty.java | 32 + .../carbondata/datamap/DataMapProvider.java | 105 ++ .../datamap/IndexDataMapProvider.java | 116 +++ .../datamap/PreAggregateDataMapProvider.java | 92 ++ .../datamap/TimeseriesDataMapProvider.java | 49 + .../scala/org/apache/spark/sql/CarbonEnv.scala | 2 +- .../datamap/CarbonCreateDataMapCommand.scala | 82 +- .../datamap/CarbonDropDataMapCommand.scala | 45 +- .../CarbonAlterTableDropPartitionCommand.scala | 2 +- .../CarbonAlterTableSplitPartitionCommand.scala | 2 +- .../CreatePreAggregateTableCommand.scala | 218 ----- .../preaaggregate/PreAggregateTableHelper.scala | 202 ++++ .../preaaggregate/PreAggregateUtil.scala | 21 +- .../CarbonAlterTableAddColumnCommand.scala | 2 +- .../CarbonAlterTableDataTypeChangeCommand.scala | 2 +- .../CarbonAlterTableDropColumnCommand.scala | 2 +- .../schema/CarbonAlterTableRenameCommand.scala | 2 +- .../command/timeseries/TimeSeriesUtil.scala | 12 +- .../spark/sql/hive/CarbonFileMetastore.scala | 3 +- .../spark/sql/hive/CarbonHiveMetaStore.scala | 3 +- .../apache/spark/sql/hive/CarbonMetaStore.scala | 2 +- .../org/apache/spark/util/AlterTableUtil.scala | 10 +- .../datamap/DataMapWriterListener.java | 8 +- 75 files changed, 4087 insertions(+), 3644 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java index 7c25b2c..83cae7c 100644 --- a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java +++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java @@ -34,4 +34,8 @@ public class MalformedDataMapCommandException extends MalformedCarbonCommandExce public MalformedDataMapCommandException(String msg) { super(msg); } + + public MalformedDataMapCommandException(String msg, Throwable e) { + super(msg, e); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 4d838a8..f7c4d2c 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -1507,7 +1507,6 @@ public final class CarbonCommonConstants { public static final String CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT = "true"; - public static final String AGGREGATIONDATAMAPSCHEMA = "AggregateDataMapHandler"; /* * The total size of carbon data */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java index 5155009..41e9b56 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java @@ -212,9 +212,10 @@ public class DataMapChooser { List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) { List<DataMapTuple> tuples = new ArrayList<>(); for (TableDataMap dataMap : allDataMap) { - if (contains(dataMap.getDataMapFactory().getMeta(), columnExpressions, expressionTypes)) { + if (contains(dataMap.getIndexDataMapFactory().getMeta(), columnExpressions, expressionTypes)) + { tuples.add( - new DataMapTuple(dataMap.getDataMapFactory().getMeta().getIndexedColumns().size(), + new DataMapTuple(dataMap.getIndexDataMapFactory().getMeta().getIndexedColumns().size(), dataMap)); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java new file mode 100644 index 0000000..03c0c3e --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +public class DataMapRegistry { + private static Map<String, String> shortNameToClassName = new ConcurrentHashMap<>(); + + public static void registerDataMap(String datamapClassName, String shortName) { + Objects.requireNonNull(datamapClassName); + Objects.requireNonNull(shortName); + shortNameToClassName.put(shortName, datamapClassName); + } + + public static String getDataMapClassName(String shortName) { + Objects.requireNonNull(shortName); + return shortNameToClassName.get(shortName); + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java index ca5da1e..ab31393 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java @@ -26,12 +26,12 @@ import org.apache.carbondata.common.exceptions.MetadataProcessException; import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datamap.dev.DataMapFactory; +import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory; import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher; import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher; -import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory; +import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMapFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; import org.apache.carbondata.core.mutate.SegmentUpdateDetails; @@ -68,7 +68,7 @@ public final class DataMapStoreManager { List<TableDataMap> tableDataMaps = getAllDataMap(carbonTable); if (tableDataMaps != null) { for (TableDataMap dataMap : tableDataMaps) { - if (mapType == dataMap.getDataMapFactory().getDataMapType()) { + if (mapType == dataMap.getIndexDataMapFactory().getDataMapType()) { dataMaps.add(dataMap); } } @@ -86,8 +86,8 @@ public final class DataMapStoreManager { List<TableDataMap> dataMaps = new ArrayList<>(); if (dataMapSchemaList != null) { for (DataMapSchema dataMapSchema : dataMapSchemaList) { - if (!dataMapSchema.getClassName() - .equalsIgnoreCase(CarbonCommonConstants.AGGREGATIONDATAMAPSCHEMA)) { + if (!dataMapSchema.getClassName().equalsIgnoreCase( + DataMapProvider.PREAGGREGATE.toString())) { dataMaps.add(getDataMap(carbonTable.getAbsoluteTableIdentifier(), dataMapSchema)); } } @@ -96,13 +96,13 @@ public final class DataMapStoreManager { } /** - * It gives the default datamap of the table. Default datamap of any table is BlockletDataMap + * It gives the default datamap of the table. Default datamap of any table is BlockletIndexDataMap * * @param identifier * @return */ public TableDataMap getDefaultDataMap(AbsoluteTableIdentifier identifier) { - return getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA); + return getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA); } /** @@ -141,9 +141,26 @@ public final class DataMapStoreManager { * Return a new datamap instance and registered in the store manager. * The datamap is created using datamap name, datamap factory class and table identifier. */ - public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier, - DataMapSchema dataMapSchema) - throws MalformedDataMapCommandException { + private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier, + DataMapSchema dataMapSchema) throws MalformedDataMapCommandException { + IndexDataMapFactory indexDataMapFactory; + try { + // try to create datamap by reflection to test whether it is a valid IndexDataMapFactory class + Class<? extends IndexDataMapFactory> factoryClass = + (Class<? extends IndexDataMapFactory>) Class.forName(dataMapSchema.getClassName()); + indexDataMapFactory = factoryClass.newInstance(); + } catch (ClassNotFoundException e) { + throw new MalformedDataMapCommandException( + "DataMap '" + dataMapSchema.getClassName() + "' not found"); + } catch (Throwable e) { + throw new MetadataProcessException( + "failed to create DataMap '" + dataMapSchema.getClassName() + "'", e); + } + return registerDataMap(identifier, dataMapSchema, indexDataMapFactory); + } + + public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier, + DataMapSchema dataMapSchema, IndexDataMapFactory indexDataMapFactory) { String table = identifier.getCarbonTableIdentifier().getTableUniqueName(); // Just update the segmentRefreshMap with the table if not added. getTableSegmentRefresher(identifier); @@ -151,37 +168,19 @@ public final class DataMapStoreManager { if (tableDataMaps == null) { tableDataMaps = new ArrayList<>(); } - String dataMapName = dataMapSchema.getDataMapName(); - TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps); - if (dataMap != null && dataMap.getDataMapSchema().getDataMapName() - .equalsIgnoreCase(dataMapName)) { - throw new MalformedDataMapCommandException("Already datamap exists in that path with type " + - dataMapName); - } - try { - // try to create datamap by reflection to test whether it is a valid DataMapFactory class - Class<? extends DataMapFactory> factoryClass = - (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName()); - DataMapFactory dataMapFactory = factoryClass.newInstance(); - dataMapFactory.init(identifier, dataMapSchema); - BlockletDetailsFetcher blockletDetailsFetcher; - SegmentPropertiesFetcher segmentPropertiesFetcher = null; - if (dataMapFactory instanceof BlockletDetailsFetcher) { - blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory; - } else { - blockletDetailsFetcher = getBlockletDetailsFetcher(identifier); - } - segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher; - dataMap = new TableDataMap(identifier, dataMapSchema, dataMapFactory, blockletDetailsFetcher, - segmentPropertiesFetcher); - } catch (ClassNotFoundException e) { - throw new MalformedDataMapCommandException("DataMap class '" + - dataMapSchema.getClassName() + "' not found"); - } catch (Throwable e) { - throw new MetadataProcessException( - "failed to create DataMap instance for '" + dataMapSchema.getClassName() + "'", e); + indexDataMapFactory.init(identifier, dataMapSchema); + BlockletDetailsFetcher blockletDetailsFetcher; + SegmentPropertiesFetcher segmentPropertiesFetcher = null; + if (indexDataMapFactory instanceof BlockletDetailsFetcher) { + blockletDetailsFetcher = (BlockletDetailsFetcher) indexDataMapFactory; + } else { + blockletDetailsFetcher = getBlockletDetailsFetcher(identifier); } + segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher; + TableDataMap dataMap = new TableDataMap(identifier, dataMapSchema, indexDataMapFactory, + blockletDetailsFetcher, segmentPropertiesFetcher); + tableDataMaps.add(dataMap); allDataMaps.put(table, tableDataMaps); return dataMap; @@ -261,8 +260,8 @@ public final class DataMapStoreManager { * @return */ private BlockletDetailsFetcher getBlockletDetailsFetcher(AbsoluteTableIdentifier identifier) { - TableDataMap blockletMap = getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA); - return (BlockletDetailsFetcher) blockletMap.getDataMapFactory(); + TableDataMap blockletMap = getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA); + return (BlockletDetailsFetcher) blockletMap.getIndexDataMapFactory(); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java index d4a8a22..7f0f3f2 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java @@ -22,8 +22,8 @@ import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datamap.dev.BlockletSerializer; -import org.apache.carbondata.core.datamap.dev.DataMap; -import org.apache.carbondata.core.datamap.dev.DataMapFactory; +import org.apache.carbondata.core.datamap.dev.IndexDataMap; +import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.indexstore.Blocklet; @@ -39,7 +39,7 @@ import org.apache.carbondata.events.OperationContext; import org.apache.carbondata.events.OperationEventListener; /** - * DataMap at the table level, user can add any number of datamaps for one table. Depends + * IndexDataMap at the table level, user can add any number of datamaps for one table. Depends * on the filter condition it can prune the blocklets. */ public final class TableDataMap extends OperationEventListener { @@ -48,7 +48,7 @@ public final class TableDataMap extends OperationEventListener { private DataMapSchema dataMapSchema; - private DataMapFactory dataMapFactory; + private IndexDataMapFactory indexDataMapFactory; private BlockletDetailsFetcher blockletDetailsFetcher; @@ -58,11 +58,11 @@ public final class TableDataMap extends OperationEventListener { * It is called to initialize and load the required table datamap metadata. */ public TableDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema, - DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher, + IndexDataMapFactory indexDataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher, SegmentPropertiesFetcher segmentPropertiesFetcher) { this.identifier = identifier; this.dataMapSchema = dataMapSchema; - this.dataMapFactory = dataMapFactory; + this.indexDataMapFactory = indexDataMapFactory; this.blockletDetailsFetcher = blockletDetailsFetcher; this.segmentPropertiesFetcher = segmentPropertiesFetcher; } @@ -84,10 +84,10 @@ public final class TableDataMap extends OperationEventListener { if (filterExp == null) { pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segmentId, partitions); } else { - List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId); + List<IndexDataMap> indexDataMaps = indexDataMapFactory.getDataMaps(segmentId); segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId); - for (DataMap dataMap : dataMaps) { - pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions)); + for (IndexDataMap indexDataMap : indexDataMaps) { + pruneBlocklets.addAll(indexDataMap.prune(filterExp, segmentProperties, partitions)); } } blocklets.addAll(addSegmentId(blockletDetailsFetcher @@ -114,7 +114,7 @@ public final class TableDataMap extends OperationEventListener { public List<DataMapDistributable> toDistributable(List<String> segmentIds) throws IOException { List<DataMapDistributable> distributables = new ArrayList<>(); for (String segmentsId : segmentIds) { - List<DataMapDistributable> list = dataMapFactory.toDistributable(segmentsId); + List<DataMapDistributable> list = indexDataMapFactory.toDistributable(segmentsId); for (DataMapDistributable distributable: list) { distributable.setDataMapSchema(dataMapSchema); distributable.setSegmentId(segmentsId); @@ -137,10 +137,10 @@ public final class TableDataMap extends OperationEventListener { FilterResolverIntf filterExp, List<String> partitions) throws IOException { List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>(); List<Blocklet> blocklets = new ArrayList<>(); - List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable); - for (DataMap dataMap : dataMaps) { + List<IndexDataMap> indexDataMaps = indexDataMapFactory.getDataMaps(distributable); + for (IndexDataMap indexDataMap : indexDataMaps) { blocklets.addAll( - dataMap.prune( + indexDataMap.prune( filterExp, segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()), partitions)); @@ -149,13 +149,13 @@ public final class TableDataMap extends OperationEventListener { String writePath = identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapSchema .getDataMapName(); - if (dataMapFactory.getDataMapType() == DataMapType.FG) { + if (indexDataMapFactory.getDataMapType() == DataMapType.FG) { FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath)); } for (Blocklet blocklet : blocklets) { ExtendedBlocklet detailedBlocklet = blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegmentId()); - if (dataMapFactory.getDataMapType() == DataMapType.FG) { + if (indexDataMapFactory.getDataMapType() == DataMapType.FG) { String blockletwritePath = writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime(); detailedBlocklet.setDataMapWriterPath(blockletwritePath); @@ -173,7 +173,7 @@ public final class TableDataMap extends OperationEventListener { */ public void clear(List<String> segmentIds) { for (String segmentId: segmentIds) { - dataMapFactory.clear(segmentId); + indexDataMapFactory.clear(segmentId); } } @@ -181,19 +181,19 @@ public final class TableDataMap extends OperationEventListener { * Clears all datamap */ public void clear() { - dataMapFactory.clear(); + indexDataMapFactory.clear(); } public DataMapSchema getDataMapSchema() { return dataMapSchema; } - public DataMapFactory getDataMapFactory() { - return dataMapFactory; + public IndexDataMapFactory getIndexDataMapFactory() { + return indexDataMapFactory; } @Override public void onEvent(Event event, OperationContext opContext) throws Exception { - dataMapFactory.fireEvent(event); + indexDataMapFactory.fireEvent(event); } /** @@ -208,9 +208,9 @@ public final class TableDataMap extends OperationEventListener { throws IOException { List<String> prunedSegments = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (String segmentId : segmentIds) { - List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId); - for (DataMap dataMap : dataMaps) { - if (dataMap.isScanRequired(filterExp)) { + List<IndexDataMap> indexDataMaps = indexDataMapFactory.getDataMaps(segmentId); + for (IndexDataMap indexDataMap : indexDataMaps) { + if (indexDataMap.isScanRequired(filterExp)) { // If any one task in a given segment contains the data that means the segment need to // be scanned and we need to validate further data maps in the same segment prunedSegments.add(segmentId); http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java index bcc9bad..1af7dde 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java @@ -70,7 +70,7 @@ public abstract class AbstractDataMapWriter { /** * Add the column pages row to the datamap, order of pages is same as `indexColumns` in - * DataMapMeta returned in DataMapFactory. + * DataMapMeta returned in IndexDataMapFactory. * Implementation should copy the content of `pages` as needed, because `pages` memory * may be freed after this method returns, if using unsafe column page. */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java deleted file mode 100644 index 3fa6d75..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev; - -import java.io.IOException; -import java.util.List; - -import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.indexstore.Blocklet; -import org.apache.carbondata.core.memory.MemoryException; -import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; - -/** - * Datamap is an entity which can store and retrieve index data. - */ -public interface DataMap<T extends Blocklet> { - - /** - * It is called to load the data map to memory or to initialize it. - */ - void init(DataMapModel dataMapModel) throws MemoryException, IOException; - - /** - * Prune the datamap with filter expression and partition information. It returns the list of - * blocklets where these filters can exist. - * - * @param filterExp - * @return - */ - List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties, - List<String> partitions); - - // TODO Move this method to Abstract class - /** - * Validate whether the current segment needs to be fetching the required data - * - * @param filterExp - * @return - */ - boolean isScanRequired(FilterResolverIntf filterExp); - - /** - * Clear complete index table and release memory. - */ - void clear(); - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java deleted file mode 100644 index 0aebe9b..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev; - -import java.io.IOException; -import java.util.List; - -import org.apache.carbondata.core.datamap.DataMapDistributable; -import org.apache.carbondata.core.datamap.DataMapMeta; -import org.apache.carbondata.core.datamap.DataMapType; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; -import org.apache.carbondata.events.Event; - -/** - * Interface for datamap factory, it is responsible for creating the datamap. - */ -public interface DataMapFactory<T extends DataMap> { - - /** - * Initialization of Datamap factory with the identifier and datamap name - */ - void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema); - - /** - * Return a new write for this datamap - */ - AbstractDataMapWriter createWriter(String segmentId, String writeDirectoryPath); - - /** - * Get the datamap for segmentid - */ - List<T> getDataMaps(String segmentId) throws IOException; - - /** - * Get datamaps for distributable object. - */ - List<T> getDataMaps(DataMapDistributable distributable) throws IOException; - - /** - * Get all distributable objects of a segmentid - * @return - */ - List<DataMapDistributable> toDistributable(String segmentId); - - /** - * - * @param event - */ - void fireEvent(Event event); - - /** - * Clears datamap of the segment - */ - void clear(String segmentId); - - /** - * Clear all datamaps from memory - */ - void clear(); - - /** - * Return metadata of this datamap - */ - DataMapMeta getMeta(); - - /** - * Type of datamap whether it is FG or CG - */ - DataMapType getDataMapType(); -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java new file mode 100644 index 0000000..f86bc0a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev; + +import java.io.IOException; +import java.util.List; + +import org.apache.carbondata.core.datastore.block.SegmentProperties; +import org.apache.carbondata.core.indexstore.Blocklet; +import org.apache.carbondata.core.memory.MemoryException; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; + +/** + * Datamap is an entity which can store and retrieve index data. + */ +public interface IndexDataMap<T extends Blocklet> { + + /** + * It is called to load the data map to memory or to initialize it. + */ + void init(DataMapModel dataMapModel) throws MemoryException, IOException; + + /** + * Prune the datamap with filter expression and partition information. It returns the list of + * blocklets where these filters can exist. + * + * @param filterExp + * @return + */ + List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties, + List<String> partitions); + + // TODO Move this method to Abstract class + /** + * Validate whether the current segment needs to be fetching the required data + * + * @param filterExp + * @return + */ + boolean isScanRequired(FilterResolverIntf filterExp); + + /** + * Clear complete index table and release memory. + */ + void clear(); + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java new file mode 100644 index 0000000..0bab104 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev; + +import java.io.IOException; +import java.util.List; + +import org.apache.carbondata.core.datamap.DataMapDistributable; +import org.apache.carbondata.core.datamap.DataMapMeta; +import org.apache.carbondata.core.datamap.DataMapType; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; +import org.apache.carbondata.events.Event; + +/** + * Interface for datamap factory, it is responsible for creating the datamap. + */ +public interface IndexDataMapFactory<T extends IndexDataMap> { + + /** + * Initialization of Datamap factory with the identifier and datamap name + */ + void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema); + + /** + * Return a new write for this datamap + */ + AbstractDataMapWriter createWriter(String segmentId, String writeDirectoryPath); + + /** + * Get the datamap for segmentid + */ + List<T> getDataMaps(String segmentId) throws IOException; + + /** + * Get datamaps for distributable object. + */ + List<T> getDataMaps(DataMapDistributable distributable) throws IOException; + + /** + * Get all distributable objects of a segmentid + * @return + */ + List<DataMapDistributable> toDistributable(String segmentId); + + /** + * + * @param event + */ + void fireEvent(Event event); + + /** + * Clears datamap of the segment + */ + void clear(String segmentId); + + /** + * Clear all datamaps from memory + */ + void clear(); + + /** + * Return metadata of this datamap + */ + DataMapMeta getMeta(); + + /** + * Type of datamap whether it is FG or CG + */ + DataMapType getDataMapType(); +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java deleted file mode 100644 index d79d0c6..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev.cgdatamap; - -import org.apache.carbondata.core.datamap.dev.DataMap; -import org.apache.carbondata.core.indexstore.Blocklet; - -public abstract class AbstractCoarseGrainDataMap implements DataMap<Blocklet> { - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java deleted file mode 100644 index 9789992..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev.cgdatamap; - -import org.apache.carbondata.core.datamap.DataMapType; -import org.apache.carbondata.core.datamap.dev.DataMapFactory; - -/** - * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap. - * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the - * information of block and blocklet. - * 3. Based on the splits scanrdd schedule the tasks. - */ -public abstract class AbstractCoarseGrainDataMapFactory - implements DataMapFactory<AbstractCoarseGrainDataMap> { - - @Override public DataMapType getDataMapType() { - return DataMapType.CG; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java new file mode 100644 index 0000000..df9d4e8 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev.cgdatamap; + +import org.apache.carbondata.core.datamap.dev.IndexDataMap; +import org.apache.carbondata.core.indexstore.Blocklet; + +public abstract class AbstractCoarseGrainIndexDataMap implements IndexDataMap<Blocklet> { + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java new file mode 100644 index 0000000..037c32e --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev.cgdatamap; + +import org.apache.carbondata.core.datamap.DataMapType; +import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory; + +/** + * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap. + * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the + * information of block and blocklet. + * 3. Based on the splits scanrdd schedule the tasks. + */ +public abstract class AbstractCoarseGrainIndexDataMapFactory + implements IndexDataMapFactory<AbstractCoarseGrainIndexDataMap> { + + @Override public DataMapType getDataMapType() { + return DataMapType.CG; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java index a66d31b..695f653 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java @@ -81,6 +81,6 @@ public class DataMapExprWrapperImpl implements DataMapExprWrapper { } @Override public DataMapType getDataMapType() { - return dataMap.getDataMapFactory().getDataMapType(); + return dataMap.getIndexDataMapFactory().getDataMapType(); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java deleted file mode 100644 index 310fb3b..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev.fgdatamap; - -import org.apache.carbondata.core.datamap.dev.DataMap; -import org.apache.carbondata.core.indexstore.FineGrainBlocklet; - -public abstract class AbstractFineGrainDataMap implements DataMap<FineGrainBlocklet> { - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java deleted file mode 100644 index 1ca7fc3..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datamap.dev.fgdatamap; - -import org.apache.carbondata.core.datamap.DataMapType; -import org.apache.carbondata.core.datamap.dev.DataMapFactory; - -/** - * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap. - * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the - * information of block, blocklet, page and rowids information as well. - * 3. The pruned blocklets are internally wriitten to file and returns only the block , - * blocklet and filepath information as part of Splits. - * 4. Based on the splits scanrdd schedule the tasks. - * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the - * bitset if exists. And pass this bitset as input to it. - */ -public abstract class AbstractFineGrainDataMapFactory - implements DataMapFactory<AbstractFineGrainDataMap> { - - @Override public DataMapType getDataMapType() { - return DataMapType.FG; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java new file mode 100644 index 0000000..ea536b9 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev.fgdatamap; + +import org.apache.carbondata.core.datamap.dev.IndexDataMap; +import org.apache.carbondata.core.indexstore.FineGrainBlocklet; + +public abstract class AbstractFineGrainIndexDataMap implements IndexDataMap<FineGrainBlocklet> { + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java new file mode 100644 index 0000000..762c233 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datamap.dev.fgdatamap; + +import org.apache.carbondata.core.datamap.DataMapType; +import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory; + +/** + * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap. + * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the + * information of block, blocklet, page and rowids information as well. + * 3. The pruned blocklets are internally wriitten to file and returns only the block , + * blocklet and filepath information as part of Splits. + * 4. Based on the splits scanrdd schedule the tasks. + * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the + * bitset if exists. And pass this bitset as input to it. + */ +public abstract class AbstractFineGrainIndexDataMapFactory + implements IndexDataMapFactory<AbstractFineGrainIndexDataMap> { + + @Override public DataMapType getDataMapType() { + return DataMapType.FG; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/2117c077/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java index 7598961..c175378 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java @@ -29,8 +29,8 @@ import org.apache.carbondata.core.cache.Cache; import org.apache.carbondata.core.cache.CarbonLRUCache; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; -import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap; import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapModel; +import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMap; import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.PartitionMapFileStore; @@ -41,7 +41,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; * blocks */ public class BlockletDataMapIndexStore - implements Cache<TableBlockIndexUniqueIdentifier, BlockletDataMap> { + implements Cache<TableBlockIndexUniqueIdentifier, BlockletIndexDataMap> { private static final LogService LOGGER = LogServiceFactory.getLogService(BlockletDataMapIndexStore.class.getName()); /** @@ -68,10 +68,10 @@ public class BlockletDataMapIndexStore } @Override - public BlockletDataMap get(TableBlockIndexUniqueIdentifier identifier) + public BlockletIndexDataMap get(TableBlockIndexUniqueIdentifier identifier) throws IOException { String lruCacheKey = identifier.getUniqueTableSegmentIdentifier(); - BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey); + BlockletIndexDataMap dataMap = (BlockletIndexDataMap) lruCache.get(lruCacheKey); if (dataMap == null) { try { String segmentPath = CarbonTablePath.getSegmentPath( @@ -99,14 +99,15 @@ public class BlockletDataMapIndexStore } @Override - public List<BlockletDataMap> getAll( + public List<BlockletIndexDataMap> getAll( List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException { - List<BlockletDataMap> blockletDataMaps = new ArrayList<>(tableSegmentUniqueIdentifiers.size()); + List<BlockletIndexDataMap> blockletDataMaps = + new ArrayList<>(tableSegmentUniqueIdentifiers.size()); List<TableBlockIndexUniqueIdentifier> missedIdentifiers = new ArrayList<>(); // Get the datamaps for each indexfile from cache. try { for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) { - BlockletDataMap ifPresent = getIfPresent(identifier); + BlockletIndexDataMap ifPresent = getIfPresent(identifier); if (ifPresent != null) { blockletDataMaps.add(ifPresent); } else { @@ -145,7 +146,7 @@ public class BlockletDataMapIndexStore } } } catch (Throwable e) { - for (BlockletDataMap dataMap : blockletDataMaps) { + for (BlockletIndexDataMap dataMap : blockletDataMaps) { dataMap.clear(); } throw new IOException("Problem in loading segment blocks.", e); @@ -160,9 +161,9 @@ public class BlockletDataMapIndexStore * @return */ @Override - public BlockletDataMap getIfPresent( + public BlockletIndexDataMap getIfPresent( TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) { - return (BlockletDataMap) lruCache.get( + return (BlockletIndexDataMap) lruCache.get( tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier()); } @@ -185,7 +186,7 @@ public class BlockletDataMapIndexStore * @return map of taks id to segment mapping * @throws IOException */ - private BlockletDataMap loadAndGetDataMap( + private BlockletIndexDataMap loadAndGetDataMap( TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore, PartitionMapFileStore partitionFileStore, @@ -197,9 +198,9 @@ public class BlockletDataMapIndexStore if (lock == null) { lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier); } - BlockletDataMap dataMap; + BlockletIndexDataMap dataMap; synchronized (lock) { - dataMap = new BlockletDataMap(); + dataMap = new BlockletIndexDataMap(); dataMap.init(new BlockletDataMapModel(identifier.getFilePath(), indexFileStore.getFileData(identifier.getCarbonIndexFileName()), partitionFileStore.getPartitions(identifier.getCarbonIndexFileName()), @@ -235,9 +236,9 @@ public class BlockletDataMapIndexStore @Override public void clearAccessCount( List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) { - for (TableBlockIndexUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) { - BlockletDataMap cacheable = - (BlockletDataMap) lruCache.get(segmentUniqueIdentifier.getUniqueTableSegmentIdentifier()); + for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) { + BlockletIndexDataMap cacheable = + (BlockletIndexDataMap) lruCache.get(identifier.getUniqueTableSegmentIdentifier()); cacheable.clear(); } }
