[21/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
index c8a6b1d..184bf1b 100644
---
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
+++
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
@@ -28,7 +28,7 @@ import
org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{SegmentStatus,
SegmentStatusManager}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.OperationContext
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil,
CompactionType}
@@ -74,7 +74,7 @@ class AggregateDataMapCompactor(carbonLoadModel:
CarbonLoadModel,
"true")
loadCommand.processData(sqlContext.sparkSession)
val newLoadMetaDataDetails = SegmentStatusManager.readLoadMetadata(
- carbonTable.getMetaDataFilepath, uuid)
+ carbonTable.getMetadataPath, uuid)
val updatedLoadMetaDataDetails = newLoadMetaDataDetails collect {
case load if loadMetaDataDetails.contains(load) =>
load.setMergedLoadName(mergedLoadName)
@@ -83,11 +83,8 @@ class AggregateDataMapCompactor(carbonLoadModel:
CarbonLoadModel,
load
case other => other
}
-val carbonTablePath = CarbonStorePath
-
.getCarbonTablePath(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-.getAbsoluteTableIdentifier)
-SegmentStatusManager
-
.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePathWithUUID(uuid),
+SegmentStatusManager.writeLoadDetailsIntoFile(
+ CarbonTablePath.getTableStatusFilePathWithUUID(uuid),
updatedLoadMetaDataDetails)
carbonLoadModel.setLoadMetadataDetails(updatedLoadMetaDataDetails.toList.asJava)
} finally {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 6f08154..8d3110a 100644
---
a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -59,7 +59,7 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.scan.partition.PartitionUtil
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.FailureCauses
@@ -72,7 +72,7 @@ import
org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonData
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil,
CarbonLoaderUtil}
import org.apache.carbondata.spark.{DataLoadResultImpl, PartitionFactory, _}
import org.apache.carbondata.spark.load._
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil,
DataLoadingUtil, Util}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
/**
* This is the factory class which can create different RDD depends on user
needs.
@@ -127,7 +127,7 @@ object CarbonDataRDDFactory {
LOGGER.error("Not able to acquire the compaction lock for table " +
s"${ carbonLoadModel.getDatabaseName }.${
carbonLoadModel.getTableName }")
CarbonCompactionUtil
- .createCompactionRequiredFile(carbonTable.getMetaDataFilepath,
compactionType)
+ .createCompactionRequiredFile(carbonTable.getMetadataPath,
compactionType)
// throw exception only in case of DDL trigger.
if (compactionModel.i
[12/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 447ab46..547ecaa 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,12 +35,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import
org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -73,7 +72,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends
RowLevelFilterExecut
comparator =
Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
ifDefaultValueMatchesFilter();
-if (isDimensionPresentInCurrentBlock[0] == true) {
+if (isDimensionPresentInCurrentBlock[0]) {
isNaturalSorted =
dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
&& dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
}
@@ -120,11 +119,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl
extends RowLevelFilterExecut
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] ||
isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
-minValue = blockMinValue[measureBlocksIndex[0] +
lastDimensionColOrdinal];
+minValue = blockMinValue[measureChunkIndex[0] +
lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType());
} else {
-minValue = blockMinValue[dimensionBlocksIndex[0]];
+minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -170,67 +169,69 @@ public class RowLevelRangeLessThanFiterExecuterImpl
extends RowLevelFilterExecut
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean
useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks
rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] &&
!isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
-
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+
.createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
-
segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-blockChunkHolder.getDimensionRawDataChunk()[blockIndex] =
blockChunkHolder.getDataBlock()
-.getDimensionC
[33/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row"
Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort
temp row"
This reverts commit de92ea9a123b17d903f2d1d4662299315c792954.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/78aa2cc3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/78aa2cc3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/78aa2cc3
Branch: refs/heads/carbonstore-rebase5
Commit: 78aa2cc3d1e52bf940138538a1f8c0bdfac666bc
Parents: 22bb333
Author: Jacky Li
Authored: Sat Feb 10 20:11:25 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:35 2018 +0800
--
.../carbondata/core/util/NonDictionaryUtil.java | 67 ++-
.../presto/util/CarbonDataStoreCreator.scala| 1 +
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../loading/row/IntermediateSortTempRow.java| 117 -
.../loading/sort/SortStepRowHandler.java| 466 ---
.../loading/sort/SortStepRowUtil.java | 103
.../sort/unsafe/UnsafeCarbonRowPage.java| 331 +++--
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 ++-
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 +++
.../UnsafeRowComparatorForNormalDims.java | 59 ---
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 --
.../merger/UnsafeIntermediateFileMerger.java| 118 -
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 +
.../sort/sortdata/IntermediateFileMerger.java | 95 +++-
.../IntermediateSortTempRowComparator.java | 73 ---
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94
.../sortdata/RowComparatorForNormalDims.java| 62 +++
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +++-
.../sort/sortdata/SortTempFileChunkHolder.java | 174 +--
.../sort/sortdata/TableFieldStat.java | 176 ---
28 files changed, 1294 insertions(+), 1186 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/78aa2cc3/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index fca1244..d6ecfbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,26 +82,18 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required dictionary Dimension from obj []
+ * Method to get the required Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static int getDictDimension(int index, Object[] row) {
-int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+ public static Integer getDimension(int index, Object[] row) {
+
+Integer[] dimensions = (Integer[])
row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+
return dimensions[index];
- }
- /**
- * Method to get the required non-dictionary & complex from 3-parted row
- * @param index
- * @param row
- * @return
- */
- public static byte[] getNoDictOrComplex(int index, Object[] row) {
-byte[][] nonDictArray = (byte[][])
row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-return nonDictArray[index];
}
/**
@@ -116,11 +108,60 @@ public class NonDictionaryUtil {
return measures[index];
}
+ public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
+
+return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+ }
+
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][]
byteBufferArr,
Object[] measureArray) {
+
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
+
+ /**
+ * This method will extract the single dimension from the complete high card
dims byte[].+ *
+ * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
+ *
+ * @param highCardArr
+ * @param index
+ * @param highCardinalityCount
+ * @param outBuffer
+ */
+ public static void extractSingleHighCardDims(byte[] highCardArr, int index,
+ int highCardin
[36/50] [abbrv] carbondata git commit: [HotFix][CheckStyle] Fix import related checkstyle
[HotFix][CheckStyle] Fix import related checkstyle This closes #1952 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/67ca605e Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/67ca605e Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/67ca605e Branch: refs/heads/carbonstore-rebase5 Commit: 67ca605eaaf5705aebdb611c71f59bf77508ffb7 Parents: d099053 Author: xuchuanyin Authored: Thu Feb 8 15:39:45 2018 +0800 Committer: Jacky Li Committed: Fri Mar 2 15:52:35 2018 +0800 -- .../core/indexstore/blockletindex/BlockletDataRefNode.java | 2 +- .../org/apache/carbondata/core/memory/HeapMemoryAllocator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/67ca605e/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java index b8fd6ff..50862a7 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java @@ -33,8 +33,8 @@ import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReade import org.apache.carbondata.core.indexstore.BlockletDetailInfo; import org.apache.carbondata.core.indexstore.FineGrainBlocklet; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; -import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex; +import org.apache.carbondata.core.util.BitSetGroup; /** * wrapper for blocklet data map data http://git-wip-us.apache.org/repos/asf/carbondata/blob/67ca605e/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java index 53cbb1d..242995b 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java @@ -17,11 +17,11 @@ package org.apache.carbondata.core.memory; -import javax.annotation.concurrent.GuardedBy; import java.lang.ref.WeakReference; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; +import javax.annotation.concurrent.GuardedBy; import org.apache.carbondata.core.util.CarbonProperties;
[35/50] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
Pick up the no-sort fields in the row and pack them as bytes array and skip
parsing them during merge sort to reduce CPU consumption
This closes #1792
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d115c479
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d115c479
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d115c479
Branch: refs/heads/carbonstore-rebase5
Commit: d115c479a6b4b859c834bd7a633da644e21e4abc
Parents: 67ca605
Author: xuchuanyin
Authored: Thu Feb 8 14:35:14 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:35 2018 +0800
--
.../carbondata/core/util/NonDictionaryUtil.java | 67 +--
.../presto/util/CarbonDataStoreCreator.scala| 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../loading/row/IntermediateSortTempRow.java| 117 +
.../loading/sort/SortStepRowHandler.java| 466 +++
.../loading/sort/SortStepRowUtil.java | 103
.../sort/unsafe/UnsafeCarbonRowPage.java| 331 ++---
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 +--
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 ---
.../UnsafeRowComparatorForNormalDims.java | 59 +++
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 ++
.../merger/UnsafeIntermediateFileMerger.java| 118 +
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 -
.../sort/sortdata/IntermediateFileMerger.java | 95 +---
.../IntermediateSortTempRowComparator.java | 73 +++
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94
.../sortdata/RowComparatorForNormalDims.java| 62 ---
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +---
.../sort/sortdata/SortTempFileChunkHolder.java | 174 ++-
.../sort/sortdata/TableFieldStat.java | 176 +++
28 files changed, 1186 insertions(+), 1294 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d115c479/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index d6ecfbc..fca1244 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,18 +82,26 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required Dimension from obj []
+ * Method to get the required dictionary Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static Integer getDimension(int index, Object[] row) {
-
-Integer[] dimensions = (Integer[])
row[WriteStepRowUtil.DICTIONARY_DIMENSION];
-
+ public static int getDictDimension(int index, Object[] row) {
+int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
return dimensions[index];
+ }
+ /**
+ * Method to get the required non-dictionary & complex from 3-parted row
+ * @param index
+ * @param row
+ * @return
+ */
+ public static byte[] getNoDictOrComplex(int index, Object[] row) {
+byte[][] nonDictArray = (byte[][])
row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+return nonDictArray[index];
}
/**
@@ -108,60 +116,11 @@ public class NonDictionaryUtil {
return measures[index];
}
- public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
-
-return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
- }
-
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][]
byteBufferArr,
Object[] measureArray) {
-
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
-
- /**
- * This method will extract the single dimension from the complete high card
dims byte[].+ *
- * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
- *
- * @param highCardArr
- * @param index
- * @param highCardinalityCount
- * @param outBuffer
- */
- public static v
[19/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
[CARBONDATA-2099] Refactor query scan process to improve readability Unified concepts in scan process flow: 1.QueryModel contains all parameter for scan, it is created by API in CarbonTable. (In future, CarbonTable will be the entry point for various table operations) 2.Use term ColumnChunk to represent one column in one blocklet, and use ChunkIndex in reader to read specified column chunk 3.Use term ColumnPage to represent one page in one ColumnChunk 4.QueryColumn => ProjectionColumn, indicating it is for projection This closes #1874 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/636eb799 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/636eb799 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/636eb799 Branch: refs/heads/carbonstore-rebase5 Commit: 636eb799441ad977194574ad9a4d8ef552f28a38 Parents: 6944dd4 Author: Jacky Li Authored: Tue Jan 30 21:24:04 2018 +0800 Committer: Jacky Li Committed: Fri Mar 2 15:51:54 2018 +0800 -- .../dictionary/AbstractDictionaryCache.java | 3 +- .../cache/dictionary/DictionaryCacheLoader.java | 7 +- .../dictionary/DictionaryCacheLoaderImpl.java | 11 +- .../core/datastore/BTreeBuilderInfo.java| 6 - .../carbondata/core/datastore/DataRefNode.java | 81 +-- .../carbondata/core/datastore/FileHolder.java | 118 .../carbondata/core/datastore/FileReader.java | 114 +++ .../core/datastore/block/SegmentProperties.java | 50 +- .../chunk/DimensionColumnDataChunk.java | 116 --- .../datastore/chunk/DimensionColumnPage.java| 111 +++ .../chunk/impl/AbstractDimensionColumnPage.java | 89 +++ .../chunk/impl/AbstractDimensionDataChunk.java | 95 --- .../impl/ColumnGroupDimensionColumnPage.java| 194 ++ .../impl/ColumnGroupDimensionDataChunk.java | 194 -- .../chunk/impl/DimensionRawColumnChunk.java | 46 +- .../impl/FixedLengthDimensionColumnPage.java| 163 + .../impl/FixedLengthDimensionDataChunk.java | 163 - .../chunk/impl/MeasureRawColumnChunk.java | 26 +- .../impl/VariableLengthDimensionColumnPage.java | 133 .../impl/VariableLengthDimensionDataChunk.java | 140 .../reader/DimensionColumnChunkReader.java | 14 +- .../chunk/reader/MeasureColumnChunkReader.java | 12 +- .../AbstractChunkReaderV2V3Format.java | 34 +- ...mpressedDimensionChunkFileBasedReaderV1.java | 38 +- ...mpressedDimensionChunkFileBasedReaderV2.java | 30 +- ...essedDimChunkFileBasedPageLevelReaderV3.java | 11 +- ...mpressedDimensionChunkFileBasedReaderV3.java | 49 +- .../AbstractMeasureChunkReaderV2V3Format.java | 42 +- ...CompressedMeasureChunkFileBasedReaderV1.java | 16 +- ...CompressedMeasureChunkFileBasedReaderV2.java | 24 +- ...CompressedMeasureChunkFileBasedReaderV3.java | 45 +- ...essedMsrChunkFileBasedPageLevelReaderV3.java | 8 +- .../chunk/store/ColumnPageWrapper.java | 30 +- .../chunk/store/DimensionDataChunkStore.java| 8 +- .../SafeFixedLengthDimensionDataChunkStore.java | 6 +- ...feVariableLengthDimensionDataChunkStore.java | 8 +- ...nsafeFixedLengthDimensionDataChunkStore.java | 10 +- ...afeVariableLengthDimesionDataChunkStore.java | 10 +- .../datastore/columnar/ColumnGroupModel.java| 26 - .../core/datastore/impl/DFSFileHolderImpl.java | 166 - .../core/datastore/impl/DFSFileReaderImpl.java | 155 .../datastore/impl/DefaultFileTypeProvider.java | 16 +- .../core/datastore/impl/FileFactory.java| 4 +- .../core/datastore/impl/FileHolderImpl.java | 224 -- .../core/datastore/impl/FileReaderImpl.java | 215 ++ .../core/datastore/impl/FileTypeInerface.java | 4 +- .../impl/btree/AbstractBTreeLeafNode.java | 60 +- .../impl/btree/BTreeDataRefNodeFinder.java | 6 +- .../datastore/impl/btree/BTreeNonLeafNode.java | 52 +- .../impl/btree/BlockBTreeLeafNode.java | 6 +- .../impl/btree/BlockletBTreeLeafNode.java | 46 +- .../page/encoding/EncodingFactory.java | 8 +- .../server/NonSecureDictionaryServer.java | 1 - .../core/indexstore/BlockletDetailInfo.java | 4 - .../blockletindex/BlockletDataRefNode.java | 228 ++ .../BlockletDataRefNodeWrapper.java | 241 --- .../indexstore/blockletindex/IndexWrapper.java | 2 +- .../blockletindex/SegmentIndexFileStore.java| 7 +- .../core/memory/HeapMemoryAllocator.java| 2 +- .../core/metadata/blocklet/SegmentInfo.java | 19 - .../core/metadata/schema/table/CarbonTable.java | 130 +++- .../schema/table/RelationIdentifier.java| 16 - .../core/metadata/schema/table/TableInfo.java | 6 +- .../schema/table/column/CarbonColumn.java | 2 +- .../schema/table/column/CarbonDimension.java| 12 - .../core/mutate/CarbonUpdateUtil.java | 20 +- .
[04/50] [abbrv] carbondata git commit: [CARBONDATA-1968] Add external table support
[CARBONDATA-1968] Add external table support
This PR adds support for creating external table with existing carbondata
files, using Hive syntax.
CREATE EXTERNAL TABLE tableName STORED BY 'carbondata' LOCATION 'path'
This closes #1749
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2529e477
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2529e477
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2529e477
Branch: refs/heads/carbonstore-rebase5
Commit: 2529e477032c623563087494060ecfdd34df5579
Parents: 2e28c15
Author: Jacky Li
Authored: Tue Jan 2 23:46:14 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:51:53 2018 +0800
--
.../core/metadata/schema/table/CarbonTable.java | 9 ++
.../createTable/TestCreateExternalTable.scala | 91
.../TestDataWithDicExcludeAndInclude.scala | 10 ---
.../command/table/CarbonDropTableCommand.scala | 5 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 64 +-
5 files changed, 147 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2529e477/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 09ff440..6036569 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -785,6 +785,15 @@ public class CarbonTable implements Serializable {
&& !tableInfo.getParentRelationIdentifiers().isEmpty();
}
+ /**
+ * Return true if this is an external table (table with property
"_external"="true", this is
+ * an internal table property set during table creation)
+ */
+ public boolean isExternalTable() {
+String external =
tableInfo.getFactTable().getTableProperties().get("_external");
+return external != null && external.equalsIgnoreCase("true");
+ }
+
public long size() throws IOException {
Map dataIndexSize = CarbonUtil.calculateDataIndexSize(this);
Long dataSize =
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2529e477/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
--
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
new file mode 100644
index 000..67370eb
--- /dev/null
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import java.io.File
+
+import org.apache.spark.sql.{AnalysisException, CarbonEnv}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
+
+ var originDataPath: String = _
+
+ override def beforeAll(): Unit = {
+sql("DROP TABLE IF EXISTS origin")
+// create carbon table and insert data
+sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'")
+sql("INSERT INTO origin select 100,'spark'")
+sql("INSERT INTO origin select 200,'hive'")
+originDataPath = s"$storeLocation/origin"
+ }
+
+ override def afterAll(): Unit = {
+sql("DROP TABLE IF EXISTS origin")
+ }
+
+ test("create external table with existing files") {
+assert(new
[06/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove partitionId in CarbonTablePath
[CARBONDATA-1992] Remove partitionId in CarbonTablePath
In CarbonTablePath, there is a deprecated partition id which is always 0, it
should be removed to avoid confusion.
This closes #1765
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2e28c156
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2e28c156
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2e28c156
Branch: refs/heads/carbonstore-rebase5
Commit: 2e28c15681066bad85dab0104307f927fb777a63
Parents: 7f7ea4d
Author: Jacky Li
Authored: Sat Jan 6 20:28:44 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:51:53 2018 +0800
--
.../core/metadata/PartitionMapFileStore.java| 0
.../core/mutate/CarbonUpdateUtil.java | 8 +-
.../core/statusmanager/LoadMetadataDetails.java | 1 +
.../SegmentUpdateStatusManager.java | 6 +-
.../apache/carbondata/core/util/CarbonUtil.java | 6 +-
.../core/util/path/CarbonTablePath.java | 55 ---
.../CarbonFormatDirectoryStructureTest.java | 4 +-
.../hadoop/api/CarbonTableInputFormat.java | 2 +-
.../streaming/CarbonStreamRecordWriter.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 1 -
.../presto/util/CarbonDataStoreCreator.scala| 1 -
.../dataload/TestLoadDataGeneral.scala | 2 +-
.../InsertIntoCarbonTableTestCase.scala | 4 +-
.../dataload/TestBatchSortDataLoad.scala| 3 +-
.../dataload/TestDataLoadWithFileName.scala | 2 +-
.../dataload/TestGlobalSortDataLoad.scala | 4 +-
.../testsuite/datamap/TestDataMapCommand.scala | 34 ++--
.../TestDataLoadingForPartitionTable.scala | 3 +-
.../load/DataLoadProcessBuilderOnSpark.scala| 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 2 +-
.../spark/rdd/AlterTableLoadPartitionRDD.scala | 154 +++
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 11 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala| 25 ++-
.../org/apache/spark/util/PartitionUtils.scala | 5 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 5 +-
.../datasources/CarbonFileFormat.scala | 1 -
.../partition/TestAlterPartitionTable.scala | 2 +-
.../bucketing/TableBucketingTestCase.scala | 2 +
.../loading/CarbonDataLoadConfiguration.java| 10 --
.../loading/DataLoadProcessBuilder.java | 1 -
.../loading/TableProcessingOperations.java | 3 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 4 +-
...arallelReadMergeSorterWithBucketingImpl.java | 15 +-
.../UnsafeBatchParallelReadMergeSorterImpl.java | 7 +-
...arallelReadMergeSorterWithBucketingImpl.java | 21 ++-
.../CarbonRowDataWriterProcessorStepImpl.java | 33 ++--
.../steps/DataWriterBatchProcessorStepImpl.java | 25 +--
.../steps/DataWriterProcessorStepImpl.java | 22 +--
.../processing/merger/CarbonDataMergerUtil.java | 6 +-
.../merger/CompactionResultSortProcessor.java | 4 +-
.../sort/sortdata/SortParameters.java | 16 +-
.../store/CarbonFactDataHandlerModel.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 9 +-
.../processing/util/CarbonLoaderUtil.java | 12 +-
.../processing/util/DeleteLoadFolders.java | 7 +-
.../carbondata/processing/StoreCreator.java | 1 -
.../carbondata/streaming/StreamHandoffRDD.scala | 1 -
.../streaming/StreamSinkFactory.scala | 2 +-
.../streaming/CarbonAppendableStreamSink.scala | 8 +-
49 files changed, 274 insertions(+), 282 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2e28c156/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
new file mode 100644
index 000..e69de29
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2e28c156/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index de98fa8..18eae11 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -319,9 +319,7 @@ public class CarbonUpdateUtil {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.get
[44/50] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
[CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module To make assembling JAR of store-sdk module, it should not depend on carbon-spark module This closes #1970 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c5740b19 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c5740b19 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c5740b19 Branch: refs/heads/carbonstore-rebase5 Commit: c5740b19d0e5e190e89b4ccf2e6275fd2ee0b812 Parents: bf4973c Author: Jacky Li Authored: Sun Feb 11 21:37:04 2018 +0800 Committer: Jacky Li Committed: Fri Mar 2 15:52:36 2018 +0800 -- .../java/org/apache/carbondata/common/Maps.java | 39 ++ .../org/apache/carbondata/common/Strings.java | 3 + .../exceptions/TableStatusLockException.java| 34 ++ .../sql/InvalidLoadOptionException.java | 33 + .../sql/MalformedCarbonCommandException.java| 75 +++ .../sql/MalformedDataMapCommandException.java | 37 ++ .../exceptions/sql/NoSuchDataMapException.java | 39 ++ .../carbondata/core/datamap/TableDataMap.java | 5 +- .../exception/ConcurrentOperationException.java | 50 ++ .../statusmanager/SegmentStatusManager.java | 124 .../carbondata/core/util/DeleteLoadFolders.java | 210 +++ .../preaggregate/TestPreAggCreateCommand.scala | 2 +- .../preaggregate/TestPreAggregateDrop.scala | 2 +- .../timeseries/TestTimeSeriesCreateTable.scala | 2 +- .../timeseries/TestTimeSeriesDropSuite.scala| 2 +- .../TestTimeseriesTableSelection.scala | 2 +- .../TestDataLoadWithColumnsMoreThanSchema.scala | 3 +- .../dataload/TestGlobalSortDataLoad.scala | 2 +- .../TestLoadDataWithDiffTimestampFormat.scala | 2 +- .../TestLoadDataWithFileHeaderException.scala | 11 +- ...ataWithMalformedCarbonCommandException.scala | 3 +- .../testsuite/dataload/TestLoadOptions.scala| 2 +- .../dataload/TestTableLevelBlockSize.scala | 4 +- .../testsuite/datamap/TestDataMapCommand.scala | 2 +- .../dataretention/DataRetentionTestCase.scala | 2 +- .../spark/testsuite/datetype/DateTypeTest.scala | 2 +- .../testsuite/sortcolumns/TestSortColumns.scala | 3 +- integration/spark-common/pom.xml| 5 - .../exception/ConcurrentOperationException.java | 38 -- .../MalformedCarbonCommandException.java| 69 --- .../MalformedDataMapCommandException.java | 32 - .../spark/exception/NoSuchDataMapException.java | 33 - .../org/apache/carbondata/api/CarbonStore.scala | 3 +- .../spark/CarbonColumnValidator.scala | 8 +- .../carbondata/spark/load/ValidateUtil.scala| 72 --- .../carbondata/spark/rdd/CarbonMergerRDD.scala | 6 +- .../carbondata/spark/util/CommonUtil.scala | 70 +-- .../carbondata/spark/util/DataLoadingUtil.scala | 610 --- .../spark/util/GlobalDictionaryUtil.scala | 2 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +- .../spark/rdd/CarbonDataRDDFactory.scala| 4 +- .../spark/rdd/CarbonTableCompactor.scala| 2 +- .../org/apache/spark/sql/CarbonSource.scala | 2 +- .../datamap/CarbonCreateDataMapCommand.scala| 2 +- .../datamap/CarbonDropDataMapCommand.scala | 2 +- .../CarbonAlterTableCompactionCommand.scala | 13 +- .../management/CarbonLoadDataCommand.scala | 17 +- .../CarbonProjectForDeleteCommand.scala | 2 +- .../CarbonProjectForUpdateCommand.scala | 2 +- .../command/mutation/IUDCommonUtil.scala| 2 +- .../CreatePreAggregateTableCommand.scala| 7 +- .../preaaggregate/PreAggregateUtil.scala| 2 +- .../schema/CarbonAlterTableRenameCommand.scala | 3 +- .../command/timeseries/TimeSeriesUtil.scala | 2 +- .../datasources/CarbonFileFormat.scala | 14 +- .../sql/execution/strategy/DDLStrategy.scala| 2 +- .../strategy/StreamingTableStrategy.scala | 2 +- .../execution/command/CarbonHiveCommands.scala | 2 +- .../sql/parser/CarbonSpark2SqlParser.scala | 2 +- .../spark/sql/parser/CarbonSparkSqlParser.scala | 2 +- .../org/apache/spark/util/AlterTableUtil.scala | 2 +- .../org/apache/spark/util/TableAPIUtil.scala| 2 +- .../spark/sql/hive/CarbonSessionState.scala | 3 +- .../segmentreading/TestSegmentReading.scala | 2 +- .../spark/util/AllDictionaryTestCase.scala | 4 +- .../util/ExternalColumnDictionaryTestCase.scala | 6 +- .../TestStreamingTableOperation.scala | 4 +- .../bucketing/TableBucketingTestCase.scala | 2 +- .../vectorreader/AddColumnTestCases.scala | 2 +- .../loading/model/CarbonLoadModel.java | 14 +- .../loading/model/CarbonLoadModelBuilder.java | 322 ++ .../processing/loading/model/LoadOption.java| 251 .../processing/util/CarbonLo
[45/50] [abbrv] carbondata git commit: Support generating assembling JAR for store-sdk module
Support generating assembling JAR for store-sdk module
Support generating assembling JAR for store-sdk module and remove junit
dependency
This closes #1976
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d85215a1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d85215a1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d85215a1
Branch: refs/heads/carbonstore-rebase5
Commit: d85215a134a2be2048668cc4a736871b73a6487d
Parents: fbe9778
Author: Jacky Li
Authored: Tue Feb 13 09:12:09 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:36 2018 +0800
--
common/pom.xml| 2 +
core/pom.xml | 2 +
hadoop/pom.xml| 1 +
integration/presto/pom.xml| 3 +-
integration/spark-common-cluster-test/pom.xml | 2 +-
integration/spark-common-test/pom.xml | 3 +-
integration/spark-common/pom.xml | 2 +-
integration/spark2/pom.xml| 2 +-
pom.xml | 5 +++
processing/pom.xml| 1 +
store/sdk/pom.xml | 50 +-
streaming/pom.xml | 1 -
12 files changed, 66 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 5550129..433d575 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,10 +42,12 @@
junit
junit
+ test
org.jmockit
jmockit
+ test
org.apache.hadoop
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 92c9607..824de0d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,10 +70,12 @@
org.jmockit
jmockit
+ test
junit
junit
+ test
org.apache.spark
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 2aaac99..c3964c5 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -42,6 +42,7 @@
junit
junit
+ test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index aaaf175..0abcf38 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -193,7 +193,7 @@
org.scalatest
- scalatest_2.11
+ scalatest_${scala.binary.version}
org.apache.zookeeper
@@ -330,7 +330,6 @@
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/integration/spark-common-cluster-test/pom.xml
--
diff --git a/integration/spark-common-cluster-test/pom.xml
b/integration/spark-common-cluster-test/pom.xml
index fd907a3..028da11 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -49,11 +49,11 @@
junit
junit
+ test
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/integration/spark-common-test/pom.xml
--
diff --git a/integration/spark-common-test/pom.xml
b/integration/spark-common-test/pom.xml
index 67a2317..d1c04ae 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -106,16 +106,17 @@
junit
junit
+ test
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
org.jmockit
jmockit
+ test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d85215a1/integration/spark-common/pom.xml
--
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 295d62b..16f327d 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -58,11 +58,11 @@
junit
junit
+ test
[11/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
new file mode 100644
index 000..fde4e55
--- /dev/null
+++
b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFullScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.TaskMetricsMap;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public class DataBlockIterator extends CarbonIterator> {
+
+ /**
+ * iterator which will be used to iterate over blocklets
+ */
+ private BlockletIterator blockletIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ private ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ private BlockletScanner blockletScanner;
+
+ /**
+ * batch size of result
+ */
+ private int batchSize;
+
+ private ExecutorService executorService;
+
+ private Future future;
+
+ private Future futureIo;
+
+ private BlockletScannedResult scannedResult;
+
+ private BlockExecutionInfo blockExecutionInfo;
+
+ private FileReader fileReader;
+
+ private AtomicBoolean nextBlock;
+
+ private AtomicBoolean nextRead;
+
+ public DataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileReader
fileReader,
+ int batchSize, QueryStatisticsModel queryStatisticsModel,
ExecutorService executorService) {
+this.blockExecutionInfo = blockExecutionInfo;
+this.fileReader = fileReader;
+blockletIterator = new
BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+blockExecutionInfo.getNumberOfBlockToScan());
+if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new BlockletFilterScanner(blockExecutionInfo,
queryStatisticsModel);
+} else {
+ blockletScanner = new BlockletFullScanner(blockExecutionInfo,
queryStatisticsModel);
+}
+this.scannerResultAggregator =
+ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
+this.batchSize = batchSize;
+this.executorService = executorService;
+this.nextBlock = new AtomicBoolean(false);
+this.nextRead = new AtomicBoolean(false);
+ }
+
+ @Override
+ public List next() {
+List collectedResult = null;
+if (updateScanner()) {
+ collectedResult =
this.scannerResultAggregator.collectResultInRow(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+List data = this.scannerResultAggregator
+.collectResultInRow(scannedResult, batchSize -
collectedResul
[15/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 69f5ceb..22d1df1 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -43,10 +43,9 @@ import
org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -64,8 +63,8 @@ import
org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -121,7 +120,6 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
queryProperties.queryStatisticsRecorder =
CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
-QueryUtil.resolveQueryModel(queryModel);
QueryStatistic queryStatistic = new QueryStatistic();
// sort the block info
// so block will be loaded in sorted order this will be required for
@@ -168,12 +166,12 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR,
System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
// calculating the total number of aggeragted columns
-int measureCount = queryModel.getQueryMeasures().size();
+int measureCount = queryModel.getProjectionMeasures().size();
int currentIndex = 0;
DataType[] dataTypes = new DataType[measureCount];
-for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+for (ProjectionMeasure carbonMeasure : queryModel.getProjectionMeasures())
{
// adding the data type and aggregation type of all the measure this
// can be used
// to select the aggregator
@@ -198,9 +196,11 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
queryStatistic = new QueryStatistic();
// dictionary column unique column id to dictionary mapping
// which will be used to get column actual data
-queryProperties.columnToDictionayMapping = QueryUtil
-.getDimensionDictionaryDetail(queryModel.getQueryDimension(),
-queryProperties.complexFilterDimension,
queryModel.getAbsoluteTableIdentifier(),
+queryProperties.columnToDictionayMapping =
+QueryUtil.getDimensionDictionaryDetail(
+queryModel.getProjectionDimensions(),
+queryProperties.complexFilterDimension,
+queryModel.getAbsoluteTableIdentifier(),
tableProvider);
queryStatistic
.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY,
System.currentTimeMillis());
@@ -263,8 +263,8 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
// and query will be executed based on that infos
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
- BlockletDataRefNodeWrapper dataRefNode =
- (BlockletDataRefNodeWrapper) abstractIndex.getDataRefNode();
+ BlockletDataRefNode dataRefNode =
+ (BlockletDataRefNode) abstractIndex.getDataRefNode();
blockExec
[49/50] [abbrv] carbondata git commit: [CARBONDATA-2186] Add InterfaceAudience.Internal to annotate internal interface
[CARBONDATA-2186] Add InterfaceAudience.Internal to annotate internal interface
This closes #1986
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b1bc9c79
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b1bc9c79
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b1bc9c79
Branch: refs/heads/carbonstore-rebase5
Commit: b1bc9c79a02d70cdd9bb3c7de076929f63ced3b7
Parents: d85215a
Author: Jacky Li
Authored: Tue Feb 20 11:16:53 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:37 2018 +0800
--
.../java/org/apache/carbondata/common/Maps.java | 2 +-
.../org/apache/carbondata/common/Strings.java| 2 +-
.../common/annotations/InterfaceAudience.java| 19 ++-
.../common/annotations/InterfaceStability.java | 2 +-
.../loading/model/CarbonLoadModelBuilder.java| 2 +-
.../processing/loading/model/LoadOption.java | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java | 4 +---
7 files changed, 20 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1bc9c79/common/src/main/java/org/apache/carbondata/common/Maps.java
--
diff --git a/common/src/main/java/org/apache/carbondata/common/Maps.java
b/common/src/main/java/org/apache/carbondata/common/Maps.java
index 14fc329..4e76192 100644
--- a/common/src/main/java/org/apache/carbondata/common/Maps.java
+++ b/common/src/main/java/org/apache/carbondata/common/Maps.java
@@ -21,7 +21,7 @@ import java.util.Map;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class Maps {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1bc9c79/common/src/main/java/org/apache/carbondata/common/Strings.java
--
diff --git a/common/src/main/java/org/apache/carbondata/common/Strings.java
b/common/src/main/java/org/apache/carbondata/common/Strings.java
index 08fdc3c..23c7f9f 100644
--- a/common/src/main/java/org/apache/carbondata/common/Strings.java
+++ b/common/src/main/java/org/apache/carbondata/common/Strings.java
@@ -21,7 +21,7 @@ import java.util.Objects;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class Strings {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1bc9c79/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
index fa9729d..8d214ff 100644
---
a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
+++
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
@@ -25,10 +25,10 @@ import java.lang.annotation.RetentionPolicy;
* This annotation is ported and modified from Apache Hadoop project.
*
* Annotation to inform users of a package, class or method's intended
audience.
- * Currently the audience can be {@link User}, {@link Developer}
+ * Currently the audience can be {@link User}, {@link Developer}, {@link
Internal}
*
* Public classes that are not marked with this annotation must be
- * considered by default as {@link Developer}.
+ * considered by default as {@link Internal}.
*
* External applications must only use classes that are marked {@link User}.
*
@@ -47,12 +47,21 @@ public class InterfaceAudience {
public @interface User { }
/**
- * Intended only for developers to extend interface for CarbonData project
- * For example, new Datamap implementations.
+ * Intended for developers to develop extension for Apache CarbonData project
+ * For example, "Index DataMap" to add a new index implementation, etc
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
- public @interface Developer { }
+ public @interface Developer {
+String[] value();
+ }
+
+ /**
+ * Intended only for internal usage within Apache CarbonData project.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Internal { }
private InterfaceAudience() { } // Audience can't exist on its own
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1bc9c79/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
b/co
[34/50] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d115c479/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 11b3d43..527452a 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,15 +31,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import
org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import
org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -63,21 +62,15 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
* entry count
*/
private int entryCount;
-
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -100,27 +93,21 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
private int numberOfObjectRead;
- private int nullSetWordsLength;
-
- private Comparator comparator;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator comparator;
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters
parameters) {
// set temp file
this.tempFile = tempFile;
-this.dimCnt = parameters.getDimColCount();
-this.complexCnt = parameters.getComplexDimColCount();
-this.measureCnt = parameters.getMeasureColCount();
-this.isNoDictionaryDimensionColumn =
parameters.getNoDictionaryDimnesionColumn();
-this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-
+this.tableFieldStat = new TableFieldStat(parameters);
+this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.executorService = Executors.newFixedThreadPool(1);
-this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
-comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
+comparator = new
IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -169,11 +156,17 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
+ @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+this.returnRow =
sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+this.numberOfObjectRead++;
+ } catch (IOException e) {
+throw new CarbonSortKeyAndGroupByException("Problems while reading
row", e);
+ }
}
}
@@ -207,63 +200,22 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
}
/**
- * @retu
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
index 0cb2918..099fffd 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
@@ -29,31 +29,12 @@ public class SegmentInfo implements Serializable {
private static final long serialVersionUID = -174987462709431L;
/**
- * number of column in the segment
- */
- private int numberOfColumns;
-
- /**
* cardinality of each columns
* column which is not participating in the multidimensional key cardinality
will be -1;
*/
private int[] columnCardinality;
/**
- * @return the numberOfColumns
- */
- public int getNumberOfColumns() {
-return numberOfColumns;
- }
-
- /**
- * @param numberOfColumns the numberOfColumns to set
- */
- public void setNumberOfColumns(int numberOfColumns) {
-this.numberOfColumns = numberOfColumns;
- }
-
- /**
* @return the columnCardinality
*/
public int[] getColumnCardinality() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6036569..d17d865 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -19,7 +19,13 @@ package org.apache.carbondata.core.metadata.schema.table;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -33,7 +39,10 @@ import
org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import
org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -136,10 +145,7 @@ public class CarbonTable implements Serializable {
/**
* During creation of TableInfo from hivemetastore the DataMapSchemas and
the columns
* DataTypes are not converted to the appropriate child classes.
- *
* This method will cast the same to the appropriate classes
- *
- * @param tableInfo
*/
public static void updateTableInfo(TableInfo tableInfo) {
List dataMapSchemas = new ArrayList<>();
@@ -153,8 +159,9 @@ public class CarbonTable implements Serializable {
}
tableInfo.setDataMapSchemaList(dataMapSchemas);
for (ColumnSchema columnSchema :
tableInfo.getFactTable().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(),
columnSchema.getScale()));
}
List childSchema = tableInfo.getDataMapSchemaList();
for (DataMapSchema dataMapSchema : childSchema) {
@@ -168,10 +175,11 @@ public class CarbonTable implements Serializable {
}
}
if (tableInfo.getFactTable().getBucketingInfo() != null) {
- for (ColumnSchema columnSchema : tableInfo.getFactTable()
- .getBucketingInfo().getListOfColumns()) {
-
columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
-columnSchema.getPrecision(), columnSchema.getScale()));
+ for (ColumnSchema columnSchema :
+ tableInfo.getFactTable().getBucketingInfo().getListOfColumns()) {
+columnSchema.setDataType(
+DataTypeUtil.valueOf(
+columnSchema.getDataType(), columnSchema.getPrecision(),
[47/50] [abbrv] carbondata git commit: [CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data loading
[CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data
loading
Enhance data loading performance by specifying sort column bounds
1. Add row range number during convert-process-step
2. Dispatch rows to each sorter by range number
3. Sort/Write process step can be done concurrently in each range
4. Since all sorttemp files will be written in one folders, we add range
number to the file name to distingush them
Tests added and docs updated
This closes #1953
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b1c96285
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b1c96285
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b1c96285
Branch: refs/heads/carbonstore-rebase5
Commit: b1c962855ebdcdf57a1f2531183d14f2ade2c94c
Parents: b1bc9c7
Author: xuchuanyin
Authored: Tue Feb 13 10:58:06 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:37 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/row/CarbonRow.java | 10 +-
.../ThriftWrapperSchemaConverterImpl.java | 2 +-
.../core/metadata/schema/BucketingInfo.java | 24 +-
.../core/metadata/schema/ColumnRangeInfo.java | 29 ++
.../metadata/schema/SortColumnRangeInfo.java| 83 +
docs/data-management-on-carbondata.md | 11 +
.../TestLoadDataWithSortColumnBounds.scala | 348 +++
.../carbondata/spark/rdd/CarbonScanRDD.scala| 2 +-
.../carbondata/spark/rdd/PartitionDropper.scala | 2 +-
.../spark/rdd/PartitionSplitter.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../loading/CarbonDataLoadConfiguration.java| 11 +
.../loading/DataLoadProcessBuilder.java | 77 +++-
.../loading/converter/RowConverter.java | 2 +-
.../converter/impl/RowConverterImpl.java| 5 +
.../loading/model/CarbonLoadModel.java | 14 +
.../loading/model/CarbonLoadModelBuilder.java | 1 +
.../processing/loading/model/LoadOption.java| 1 +
.../partition/impl/HashPartitionerImpl.java | 10 +-
.../partition/impl/RangePartitionerImpl.java| 71
.../partition/impl/RawRowComparator.java| 63
.../processing/loading/sort/SorterFactory.java | 16 +-
...arallelReadMergeSorterWithBucketingImpl.java | 272 ---
...allelReadMergeSorterWithColumnRangeImpl.java | 289 +++
...arallelReadMergeSorterWithBucketingImpl.java | 263 --
...allelReadMergeSorterWithColumnRangeImpl.java | 293
.../loading/sort/unsafe/UnsafeSortDataRows.java | 6 +-
.../unsafe/merger/UnsafeIntermediateMerger.java | 6 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 11 +-
.../steps/DataConverterProcessorStepImpl.java | 102 +-
...ConverterProcessorWithBucketingStepImpl.java | 161 -
.../steps/DataWriterProcessorStepImpl.java | 70 +++-
.../SingleThreadFinalSortFilesMerger.java | 3 +-
.../processing/sort/sortdata/SortDataRows.java | 11 +-
.../sortdata/SortIntermediateFileMerger.java| 6 +-
.../sort/sortdata/SortParameters.java | 10 +
.../store/CarbonFactDataHandlerColumnar.java| 6 +-
39 files changed, 1558 insertions(+), 750 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1c96285/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a6bf60f..8ff8dc4 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -124,4 +124,14 @@ public final class CarbonLoadOptionConstants {
public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
= "carbon.load.skewedDataOptimization.enabled";
public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
+
+ /**
+ * field delimiter for each field in one bound
+ */
+ public static final String SORT_COLUMN_BOUNDS_FIELD_DELIMITER = ",";
+
+ /**
+ * row delimiter for each sort column bounds
+ */
+ public static final String SORT_COLUMN_BOUNDS_ROW_DELIMITER = ";";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b1c96285/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
--
diff --git
a/core/src/main/java/org/apache/carbondata
[50/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after merging master
[REBASE] Solve conflict after merging master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7540cc9c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7540cc9c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7540cc9c
Branch: refs/heads/carbonstore-rebase5
Commit: 7540cc9cab404c06eaf5c92da77ef7e36f9182b5
Parents: 8a9dd8b
Author: Jacky Li
Authored: Tue Feb 27 11:26:30 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 16:00:28 2018 +0800
--
.../carbondata/core/datamap/dev/DataMap.java| 9 +-
.../core/datamap/dev/DataMapFactory.java| 2 +-
.../exception/ConcurrentOperationException.java | 16 +-
.../core/indexstore/BlockletDetailsFetcher.java | 3 +-
.../blockletindex/BlockletDataMap.java | 3 +-
.../blockletindex/SegmentIndexFileStore.java| 2 -
.../core/metadata/PartitionMapFileStore.java| 0
.../scan/executor/util/RestructureUtil.java | 6 +-
.../statusmanager/SegmentStatusManager.java | 10 +-
.../SegmentUpdateStatusManager.java | 7 +-
.../CarbonStreamSparkStreamingExample.scala | 14 +-
...CarbonStructuredStreamingWithRowParser.scala | 8 +-
.../hadoop/api/CarbonTableInputFormat.java | 5 +-
.../preaggregate/TestPreAggCreateCommand.scala | 2 +-
.../TestInsertAndOtherCommandConcurrent.scala | 2 +-
.../StandardPartitionGlobalSortTestCase.scala | 2 +-
.../exception/ProcessMetaDataException.java | 2 +
.../org/apache/carbondata/api/CarbonStore.scala | 6 +-
.../carbondata/spark/load/CsvRDDHelper.scala| 157 +++
.../load/DataLoadProcessBuilderOnSpark.scala| 3 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 2 -
.../command/carbonTableSchemaCommon.scala | 6 +-
.../CarbonAlterTableCompactionCommand.scala | 3 +-
.../management/CarbonCleanFilesCommand.scala| 2 +-
.../CarbonDeleteLoadByIdCommand.scala | 2 +-
.../CarbonDeleteLoadByLoadDateCommand.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 28 ++--
.../CarbonProjectForDeleteCommand.scala | 2 +-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 3 -
.../TestStreamingTableWithRowParser.scala | 9 +-
.../vectorreader/AddColumnTestCases.scala | 1 +
.../datamap/DataMapWriterListener.java | 3 +-
.../loading/model/CarbonLoadModelBuilder.java | 34 +++-
.../processing/loading/model/LoadOption.java| 15 +-
.../processing/merger/CarbonDataMergerUtil.java | 19 ++-
.../merger/CompactionResultSortProcessor.java | 4 +-
.../merger/RowResultMergerProcessor.java| 4 +-
.../partition/spliter/RowResultProcessor.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 3 +-
store/sdk/pom.xml | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java| 8 +-
45 files changed, 305 insertions(+), 118 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7540cc9c/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 4a68286..fdeacff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -38,18 +38,13 @@ public interface DataMap {
/**
* Prune the datamap with filter expression and partition information. It
returns the list of
* blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
*/
- List prune(FilterResolverIntf filterExp, SegmentProperties
segmentProperties, List partitions);
+ List prune(FilterResolverIntf filterExp, SegmentProperties
segmentProperties,
+ List partitions);
// TODO Move this method to Abstract class
/**
* Validate whether the current segment needs to be fetching the required
data
- *
- * @param filterExp
- * @return
*/
boolean isScanRequired(FilterResolverIntf filterExp);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7540cc9c/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/Data
[08/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index f4450e3..5f8d199 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import mockit.Mock;
import mockit.MockUp;
@@ -267,8 +267,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, 1);
}
@@ -276,8 +276,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValueToTarget() {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, -1);
}
@@ -285,8 +285,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 3);
}
@@ -302,8 +302,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValueToTarget() {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 5);
}
@@ -525,23 +525,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDictionaryEncodingArray() {
-QueryDimension column1 = new QueryDimension("Column1");
-QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List encoding = new ArrayList<>();
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
-column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ProjectionDimension
+column1 = new ProjectionDimension(new CarbonDimension(column1Schema,
1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
-column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ProjectionDimension
+column2 = new ProjectionDimension(new CarbonDimension(column2Schema,
1, 1, 1, 1));
-Que
[24/50] [abbrv] carbondata git commit: [CARBONDATA-1480]Min Max Index Example for DataMap
[CARBONDATA-1480]Min Max Index Example for DataMap
Datamap Example. Implementation of Min Max Index through Datamap. And Using the
Index while prunning.
This closes #1359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6882f737
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6882f737
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6882f737
Branch: refs/heads/carbonstore-rebase5
Commit: 6882f737ebafb8a6470be638e7f4bb273e6c7292
Parents: 8da481a
Author: sounakr
Authored: Thu Sep 28 16:21:05 2017 +0530
Committer: Jacky Li
Committed: Fri Mar 2 15:52:34 2018 +0800
--
.../core/datamap/DataMapStoreManager.java | 16 +-
.../carbondata/core/datamap/TableDataMap.java | 18 +-
.../carbondata/core/datamap/dev/DataMap.java| 11 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../indexstore/SegmentPropertiesFetcher.java| 36 +++
.../blockletindex/BlockletDataMap.java | 5 +-
.../blockletindex/BlockletDataMapFactory.java | 32 ++-
datamap/examples/pom.xml| 111 ++
.../datamap/examples/BlockletMinMax.java| 41
.../datamap/examples/MinMaxDataMap.java | 143
.../datamap/examples/MinMaxDataMapFactory.java | 114 ++
.../datamap/examples/MinMaxDataWriter.java | 221 +++
.../examples/MinMaxIndexBlockDetails.java | 77 +++
.../MinMaxDataMapExample.scala | 77 +++
.../testsuite/datamap/DataMapWriterSuite.scala | 2 +-
pom.xml | 2 +
.../datamap/DataMapWriterListener.java | 4 +-
.../store/writer/AbstractFactDataWriter.java| 7 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 3 +
19 files changed, 894 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6882f737/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index d30483a..90e5fff 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -103,7 +104,7 @@ public final class DataMapStoreManager {
tableDataMaps = new ArrayList<>();
}
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
-if (dataMap != null) {
+if (dataMap != null &&
dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
throw new RuntimeException("Already datamap exists in that path with
type " + dataMapName);
}
@@ -113,12 +114,15 @@ public final class DataMapStoreManager {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory,
blockletDetailsFetcher);
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher)
blockletDetailsFetcher;
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory,
blockletDetailsFetcher,
+ segmentPropertiesFetcher);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -128,11 +132,11 @@ public final class DataMapStoreManager {
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName,
- List tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List
tableDataMaps) {
TableDataMap dataMap = null;
-for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName)) {
+for (TableDataMa
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Fix bugs in tests that will cause failure under windows env
This closes #1994
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a9dd8b2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a9dd8b2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a9dd8b2
Branch: refs/heads/carbonstore-rebase5
Commit: 8a9dd8b21907096ded81fb58e152eecc3f629bb8
Parents: b1c9628
Author: xuchuanyin
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:37 2018 +0800
--
.../carbondata/core/locks/LocalFileLock.java| 30 +++-
.../store/impl/DFSFileReaderImplUnitTest.java | 11 ---
.../store/impl/FileFactoryImplUnitTest.java | 28 +-
.../filesystem/HDFSCarbonFileTest.java | 3 +-
.../filesystem/LocalCarbonFileTest.java | 20 +
.../BooleanDataTypesInsertTest.scala| 5 ++--
.../carbondata/lcm/locks/LocalFileLockTest.java | 2 +-
.../loading/csvinput/CSVInputFormatTest.java| 1 +
8 files changed, 64 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
package org.apache.carbondata.core.locks;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
private String location;
/**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
* channel is the FileChannel of the lock file.
*/
private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
FileFactory.createNewLockFile(lockFilePath,
FileFactory.getFileType(location));
}
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
+ channel = FileChannel.open(Paths.get(lockFilePath),
StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND);
try {
fileLock = channel.tryLock();
} catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
} catch (IOException e) {
status = false;
} finally {
- if (null != fileOutputStream) {
-try {
- fileOutputStream.close();
-} catch (IOException e) {
- LOGGER.error(e.getMessage());
+ CarbonUtil.closeStreams(channel);
+
+ // deleting the lock file after releasing the lock.
+ if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a9dd8b2/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
---
a/core/src/test/java/org
[23/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
[CARBONDATA-2025] Unify all path construction through CarbonTablePath static method Refactory CarbonTablePath: 1.Remove CarbonStorePath and use CarbonTablePath only. 2.Make CarbonTablePath an utility without object creation, it can avoid creating object before using it, thus code is cleaner and GC is less. This closes #1768 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9daad358 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9daad358 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9daad358 Branch: refs/heads/carbonstore-rebase5 Commit: 9daad358a3878e8a2338bdd7c746507af0860cea Parents: 636eb79 Author: Jacky Li Authored: Wed Jan 31 16:14:27 2018 +0800 Committer: Jacky Li Committed: Fri Mar 2 15:52:32 2018 +0800 -- .../DictionaryColumnUniqueIdentifier.java | 29 +- .../dictionary/ManageDictionaryAndBTree.java| 13 +- .../core/metadata/AbsoluteTableIdentifier.java | 4 +- .../core/metadata/schema/table/CarbonTable.java | 11 +- .../core/mutate/CarbonUpdateUtil.java | 58 ++-- .../core/scan/executor/util/QueryUtil.java | 7 +- .../core/service/CarbonCommonFactory.java | 16 - .../carbondata/core/service/PathService.java| 35 --- .../core/service/impl/PathFactory.java | 50 .../statusmanager/SegmentStatusManager.java | 30 +- .../SegmentUpdateStatusManager.java | 59 ++-- .../apache/carbondata/core/util/CarbonUtil.java | 60 ++-- .../util/path/CarbonSharedDictionaryPath.java | 71 - .../core/util/path/CarbonStorePath.java | 71 - .../core/util/path/CarbonTablePath.java | 291 ++- .../dictionary/AbstractDictionaryCacheTest.java | 11 +- .../dictionary/ForwardDictionaryCacheTest.java | 6 +- .../dictionary/ReverseDictionaryCacheTest.java | 6 +- .../reader/CarbonDictionaryReaderImplTest.java | 8 - .../CarbonFormatDirectoryStructureTest.java | 18 +- .../path/CarbonFormatSharedDictionaryTest.java | 44 --- .../writer/CarbonDictionaryWriterImplTest.java | 19 +- .../CarbonBatchSparkStreamingExample.scala | 9 +- .../CarbonStructuredStreamingExample.scala | 11 +- .../hadoop/api/CarbonTableInputFormat.java | 8 +- .../streaming/CarbonStreamRecordWriter.java | 6 +- .../carbondata/hadoop/util/SchemaReader.java| 18 +- .../hadoop/test/util/StoreCreator.java | 16 +- .../presto/CarbondataRecordSetProvider.java | 7 +- .../presto/impl/CarbonTableCacheModel.java | 13 +- .../presto/impl/CarbonTableReader.java | 49 ++-- .../presto/util/CarbonDataStoreCreator.scala| 10 +- .../sdv/generated/MergeIndexTestCase.scala | 8 +- .../dataload/TestLoadDataGeneral.scala | 11 +- .../InsertIntoCarbonTableTestCase.scala | 8 +- .../createTable/TestCreateTableAsSelect.scala | 2 +- .../datacompaction/DataCompactionLockTest.scala | 6 +- .../MajorCompactionIgnoreInMinorTest.scala | 12 +- .../dataload/TestBatchSortDataLoad.scala| 5 +- .../dataload/TestDataLoadWithFileName.scala | 5 +- .../dataload/TestGlobalSortDataLoad.scala | 8 +- .../dataretention/DataRetentionTestCase.scala | 6 +- .../TestDataLoadingForPartitionTable.scala | 4 +- .../StandardPartitionTableLoadingTestCase.scala | 6 +- .../org/apache/carbondata/api/CarbonStore.scala | 4 +- .../carbondata/spark/CarbonSparkFactory.scala | 2 +- .../spark/DictionaryDetailHelper.scala | 9 +- .../spark/rdd/AlterTableAddColumnRDD.scala | 17 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 6 - .../carbondata/spark/util/CommonUtil.scala | 30 +- .../carbondata/spark/util/DataLoadingUtil.scala | 8 +- .../spark/util/GlobalDictionaryUtil.scala | 16 +- .../command/carbonTableSchemaCommon.scala | 9 +- .../org/apache/spark/util/PartitionUtils.scala | 6 +- .../spark/rdd/AggregateDataMapCompactor.scala | 11 +- .../spark/rdd/CarbonDataRDDFactory.scala| 14 +- .../spark/rdd/CarbonTableCompactor.scala| 8 +- .../CarbonAlterTableCompactionCommand.scala | 13 +- .../management/CarbonLoadDataCommand.scala | 12 +- .../management/CarbonShowLoadsCommand.scala | 2 +- .../management/RefreshCarbonTableCommand.scala | 17 +- .../command/mutation/DeleteExecution.scala | 7 +- .../CarbonAlterTableDropPartitionCommand.scala | 16 +- .../CarbonAlterTableSplitPartitionCommand.scala | 20 +- .../preaaggregate/PreAggregateUtil.scala| 35 ++- .../CarbonAlterTableAddColumnCommand.scala | 8 +- .../CarbonAlterTableDataTypeChangeCommand.scala | 9 +- .../CarbonAlterTableDropColumnCommand.scala | 6 +- .../schema/CarbonAlterTableRenameCommand.scala | 28 +- .../spark/sql/hive/CarbonFileMetastore.scala| 27
[03/50] [abbrv] carbondata git commit: [CARBONDATA-1827] S3 Carbon Implementation
[CARBONDATA-1827] S3 Carbon Implementation
1.Provide support for s3 in carbondata.
2.Added S3Example to create carbon table on s3.
3.Added S3CSVExample to load carbon table using csv from s3.
This closes #1805
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1921393f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1921393f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1921393f
Branch: refs/heads/carbonstore-rebase5
Commit: 1921393f548b65d16cc44ff1d494cfd007365d8f
Parents: 2529e47
Author: SangeetaGulia
Authored: Thu Sep 21 14:56:26 2017 +0530
Committer: Jacky Li
Committed: Fri Mar 2 15:51:53 2018 +0800
--
.../core/constants/CarbonCommonConstants.java | 21 +++
.../filesystem/AbstractDFSCarbonFile.java | 20 ++-
.../datastore/filesystem/HDFSCarbonFile.java| 5 +-
.../core/datastore/impl/FileFactory.java| 11 +-
.../core/locks/CarbonLockFactory.java | 28 ++--
.../carbondata/core/locks/S3FileLock.java | 111 +
.../carbondata/core/util/CarbonProperties.java | 3 +-
.../filesystem/HDFSCarbonFileTest.java | 8 +-
examples/spark2/pom.xml | 5 +
examples/spark2/src/main/resources/data1.csv| 11 ++
.../carbondata/examples/S3CsvExample.scala | 99 +++
.../apache/carbondata/examples/S3Example.scala | 164 +++
.../spark/rdd/NewCarbonDataLoadRDD.scala| 42 -
integration/spark2/pom.xml | 43 +
.../spark/rdd/CarbonDataRDDFactory.scala| 3 +-
.../org/apache/spark/sql/CarbonSession.scala| 3 +
16 files changed, 554 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1921393f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 0a06abc..18acc5b 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -167,6 +167,22 @@ public final class CarbonCommonConstants {
public static final String S3N_PREFIX = "s3n://";
public static final String S3A_PREFIX = "s3a://";
+ /**
+ * Access Key for s3n
+ */
+ public static final String S3N_ACCESS_KEY = "fs.s3n.awsAccessKeyId";
+ /**
+ * Secret Key for s3n
+ */
+ public static final String S3N_SECRET_KEY = "fs.s3n.awsSecretAccessKey";
+ /**
+ * Access Key for s3
+ */
+ public static final String S3_ACCESS_KEY = "fs.s3.awsAccessKeyId";
+ /**
+ * Secret Key for s3
+ */
+ public static final String S3_SECRET_KEY = "fs.s3.awsSecretAccessKey";
/**
* FS_DEFAULT_FS
@@ -941,6 +957,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_LOCK_TYPE_HDFS = "HDFSLOCK";
/**
+ * S3LOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_S3 = "S3LOCK";
+
+ /**
* Invalid filter member log string
*/
public static final String FILTER_INVALID_MEMBER =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1921393f/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 68eaa21..fd5dc40 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
/**
* LOGGER
*/
@@ -262,18 +262,28 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
@Override public DataOutputStream getDataOutputStream(String path,
FileFactory.FileType fileType,
int bufferSize, boolean append) throws IOException {
Path pt = new Path(path);
-FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());
FSDataOutputStream stream = null;
if (append) {
// appen
[30/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading"
Revert "[CARBONDATA-2023][DataLoad] Add size base block allocation in data
loading"
This reverts commit 6dd8b038fc898dbf48ad30adfc870c19eb38e3d0.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/22bb333a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/22bb333a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/22bb333a
Branch: refs/heads/carbonstore-rebase5
Commit: 22bb333a53a6f256248be519874654a794e35d17
Parents: 9062931
Author: Jacky Li
Authored: Sat Feb 10 10:34:59 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:35 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 -
.../core/datastore/block/TableBlockInfo.java| 29 --
.../carbondata/core/util/CarbonProperties.java | 11 -
docs/useful-tips-on-carbondata.md | 1 -
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 --
.../processing/util/CarbonLoaderUtil.java | 494 +++
.../processing/util/CarbonLoaderUtilTest.java | 125 -
10 files changed, 183 insertions(+), 551 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22bb333a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a6bf60f..bcfeba0 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,14 +114,4 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
- /**
- * enable block size based block allocation while loading data. By default,
carbondata assigns
- * blocks to node based on block number. If this option is set to `true`,
carbondata will
- * consider block size first and make sure that all the nodes will process
almost equal size of
- * data. This option is especially useful when you encounter skewed data.
- */
- @CarbonProperty
- public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
- = "carbon.load.skewedDataOptimization.enabled";
- public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22bb333a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c0cebe0..a7bfdba 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -100,20 +98,6 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
- /**
- * comparator to sort by block size in descending order.
- * Since each line is not exactly the same, the size of a InputSplit may
differs,
- * so we allow some deviation for these splits.
- */
- public static final Comparator DATA_SIZE_DESC_COMPARATOR =
- new Comparator() {
-@Override public int compare(Distributable o1, Distributable o2) {
- long diff =
- ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
- return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
-}
- };
-
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -450,17 +434,4 @@ public class TableBlockInfo implements Distributable,
Serializable {
public void setDataMapWriterPath(String dataMapWriterPath) {
this.dataMapWriterPath = dataMapWriterPath;
}
-
- @Override
- public String toString() {
-final StringBuilder sb = new StringBuilder("TableBlockInfo{");
-sb.append("filePath='").append(filePa
[42/50] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5740b19/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
new file mode 100644
index 000..fbb93b6
--- /dev/null
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.model;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.Maps;
+import org.apache.carbondata.common.Strings;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.constants.LoggerAction;
+import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import
org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
+import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
+import org.apache.carbondata.processing.util.TableOptionConstant;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Builder for {@link CarbonLoadModel}
+ */
+@InterfaceAudience.Developer
+public class CarbonLoadModelBuilder {
+
+ private CarbonTable table;
+
+ public CarbonLoadModelBuilder(CarbonTable table) {
+this.table = table;
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param options Load options from user input
+ * @return a new CarbonLoadModel instance
+ */
+ public CarbonLoadModel build(
+ Map options) throws InvalidLoadOptionException,
IOException {
+Map optionsFinal =
LoadOption.fillOptionWithDefaultValue(options);
+optionsFinal.put("sort_scope", "no_sort");
+if (!options.containsKey("fileheader")) {
+ List csvHeader =
table.getCreateOrderColumn(table.getTableName());
+ String[] columns = new String[csvHeader.size()];
+ for (int i = 0; i < columns.length; i++) {
+columns[i] = csvHeader.get(i).getColName();
+ }
+ optionsFinal.put("fileheader", Strings.mkString(columns, ","));
+}
+CarbonLoadModel model = new CarbonLoadModel();
+
+// we have provided 'fileheader', so it hadoopConf can be null
+build(options, optionsFinal, model, null);
+
+// set default values
+
model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+model.setUseOnePass(Boolean.parseBoolean(Maps.getOrDefault(options,
"onepass", "false")));
+model.setDictionaryServerHost(Maps.getOrDefault(options, "dicthost",
null));
+try {
+
model.setDictionaryServerPort(Integer.parseInt(Maps.getOrDefault(options,
"dictport", "-1")));
+} catch (NumberFormatException e) {
+ throw new InvalidLoadOptionException(e.getMessage());
+}
+return model;
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param options Load options from user input
+ * @param optionsFinal Load options that populated with default values for
optional options
+ * @param carbonLoadModel The output load model
+ * @param hadoopConf hadoopConf is needed to read CSV header if there
'fileheader' is not set in
+ * user provided load options
+ */
+ public void build(
+ Map options,
+ Map optionsFinal,
+ CarbonLoadModel carbonLoadModel,
+ Configuration hadoopConf) throws InvalidLoadOptionEx
[43/50] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5740b19/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 8d394db..e69de29 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import java.text.SimpleDateFormat
-import java.util
-import java.util.{Date, List, Locale}
-
-import scala.collection.{immutable, mutable}
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
-import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.hadoop.mapreduce.task.{JobContextImpl,
TaskAttemptContextImpl}
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD,
PartitionedFile}
-import org.apache.spark.sql.util.CarbonException
-import org.apache.spark.sql.util.SparkSQLUtil.sessionState
-
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.constants.{CarbonCommonConstants,
CarbonLoadOptionConstants}
-import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil,
LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import
org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema,
CarbonLoadModel}
-import org.apache.carbondata.processing.util.{CarbonLoaderUtil,
DeleteLoadFolders, TableOptionConstant}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark.LOGGER
-import org.apache.carbondata.spark.load.ValidateUtil
-import org.apache.carbondata.spark.rdd.SerializableConfiguration
-
-/**
- * the util object of data loading
- */
-object DataLoadingUtil {
-
- val LOGGER: LogService =
LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- /**
- * get data loading options and initialise default value
- */
- def getDataLoadingOptions(
- carbonProperty: CarbonProperties,
- options: immutable.Map[String, String]): mutable.Map[String, String] = {
-val optionsFinal = scala.collection.mutable.Map[String, String]()
-optionsFinal.put("delimiter", options.getOrElse("delimiter", ","))
-optionsFinal.put("quotechar", options.getOrElse("quotechar", "\""))
-optionsFinal.put("fileheader", options.getOrElse("fileheader", ""))
-optionsFinal.put("commentchar", options.getOrElse("commentchar", "#"))
-optionsFinal.put("columndict", options.getOrElse("columndict", null))
-
-optionsFinal.put("escapechar",
- CarbonLoaderUtil.getEscapeChar(options.getOrElse("escapechar", "\\")))
-
-optionsFinal.put(
- "serialization_null_format",
- options.getOrElse("ser
[25/50] [abbrv] carbondata git commit: [CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to executor for s3 implementation in cluster mode.
[CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to
executor for s3 implementation in cluster mode.
Problem : hadoopconf was not getting propagated from driver to the executor
that's why load was failing to the distributed environment.
Solution: Setting the Hadoop conf in base class CarbonRDD
How to verify this PR :
Execute the load in the cluster mode It should be a success using location s3.
This closes #1860
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8da481af
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8da481af
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8da481af
Branch: refs/heads/carbonstore-rebase5
Commit: 8da481af6b2b14e16fe289f61efec4d5ae5e0378
Parents: 9daad35
Author: Jatin
Authored: Thu Jan 25 16:53:00 2018 +0530
Committer: Jacky Li
Committed: Fri Mar 2 15:52:34 2018 +0800
--
.../spark/rdd/AlterTableAddColumnRDD.scala | 2 +-
.../spark/rdd/AlterTableDropColumnRDD.scala | 2 +-
.../spark/rdd/CarbonCleanFilesRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadRDD.scala | 2 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 2 +-
.../spark/rdd/CarbonDropTableRDD.scala | 2 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +-
.../spark/rdd/CarbonMergeFilesRDD.scala | 0
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
.../apache/carbondata/spark/rdd/CarbonRDD.scala | 32 ++--
.../spark/rdd/NewCarbonDataLoadRDD.scala| 2 +-
.../carbondata/spark/rdd/SparkDataMapJob.scala | 2 +-
.../apache/spark/rdd/DataLoadCoalescedRDD.scala | 3 +-
.../apache/spark/rdd/UpdateCoalescedRDD.scala | 2 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
16 files changed, 45 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8da481af/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index 56a66b9..7c1edea 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -50,7 +50,7 @@ class AddColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Par
class AlterTableAddColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
identifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
val lockType: String =
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8da481af/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
index 248f351..e14524e 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
@@ -48,7 +48,7 @@ class DropColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Pa
class AlterTableDropColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
carbonTableIdentifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
newColumns.zipWithIndex.map { column =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8da481af/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
index 32523d8..9936a2a 100644
---
a/integration/spark-c
[29/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap. FG Query flow as follows. 1.The user can add FG datamap to any table and implement there interfaces. 2. Any filter query which hits the table with datamap will call prune method of FGdatamap. 3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well. 4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits. 5. Based on the splits scanrdd schedule the tasks. 6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it. This closes #1471 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d0990532 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d0990532 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d0990532 Branch: refs/heads/carbonstore-rebase5 Commit: d099053285efa8267a050e299f9d8853735e4251 Parents: 6882f73 Author: ravipesala Authored: Wed Nov 15 19:48:40 2017 +0530 Committer: Jacky Li Committed: Fri Mar 2 15:52:35 2018 +0800 -- .../carbondata/core/datamap/DataMapMeta.java| 8 +- .../core/datamap/DataMapStoreManager.java | 30 +- .../carbondata/core/datamap/DataMapType.java| 21 + .../carbondata/core/datamap/TableDataMap.java | 31 +- .../core/datamap/dev/AbstractDataMapWriter.java | 110 + .../core/datamap/dev/BlockletSerializer.java| 57 +++ .../carbondata/core/datamap/dev/DataMap.java| 4 +- .../core/datamap/dev/DataMapFactory.java| 14 +- .../core/datamap/dev/DataMapWriter.java | 57 --- .../cgdatamap/AbstractCoarseGrainDataMap.java | 24 + .../AbstractCoarseGrainDataMapFactory.java | 34 ++ .../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 + .../AbstractFineGrainDataMapFactory.java| 38 ++ .../carbondata/core/datastore/DataRefNode.java | 6 + .../core/datastore/block/TableBlockInfo.java| 10 + .../impl/btree/AbstractBTreeLeafNode.java | 5 + .../datastore/impl/btree/BTreeNonLeafNode.java | 5 + .../carbondata/core/indexstore/Blocklet.java| 30 +- .../indexstore/BlockletDataMapIndexStore.java | 6 - .../core/indexstore/BlockletDetailsFetcher.java | 8 + .../core/indexstore/ExtendedBlocklet.java | 17 + .../core/indexstore/FineGrainBlocklet.java | 120 + .../blockletindex/BlockletDataMap.java | 15 +- .../blockletindex/BlockletDataMapFactory.java | 55 ++- .../blockletindex/BlockletDataRefNode.java | 27 +- .../indexstore/blockletindex/IndexWrapper.java | 18 + .../core/indexstore/schema/FilterType.java | 24 - .../executer/ExcludeFilterExecuterImpl.java | 3 + .../executer/IncludeFilterExecuterImpl.java | 3 + .../scanner/impl/BlockletFilterScanner.java | 2 + .../apache/carbondata/core/util/CarbonUtil.java | 98 + .../datamap/examples/MinMaxDataMap.java | 32 +- .../datamap/examples/MinMaxDataMapFactory.java | 49 ++- .../datamap/examples/MinMaxDataWriter.java | 36 +- .../examples/MinMaxIndexBlockDetails.java | 13 - .../carbondata/hadoop/CarbonInputSplit.java | 21 +- .../hadoop/api/CarbonTableInputFormat.java | 17 +- .../testsuite/datamap/CGDataMapTestCase.scala | 361 +++ .../testsuite/datamap/DataMapWriterSuite.scala | 43 +- .../testsuite/datamap/FGDataMapTestCase.scala | 440 +++ .../TestInsertAndOtherCommandConcurrent.scala | 21 +- .../carbondata/spark/rdd/CarbonScanRDD.scala| 7 +- .../TestStreamingTableOperation.scala | 5 +- .../datamap/DataMapWriterListener.java | 57 ++- .../store/CarbonFactDataHandlerModel.java | 10 +- .../store/writer/AbstractFactDataWriter.java| 128 +- 46 files changed, 1762 insertions(+), 382 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d0990532/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java index 7746acf..dd15ccb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java @@ -19,15 +19,15 @@ package org.apache.carbondata.core.datamap; import java.util.List; -import org.apache.carbondata.core.indexstore.schema.FilterType; +import
[02/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after rebasing master
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6944dd42
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6944dd42
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6944dd42
Branch: refs/heads/carbonstore-rebase5
Commit: 6944dd420a07f6fe694a7fed92ed256bea68d314
Parents: 1921393
Author: Jacky Li
Authored: Thu Feb 1 00:25:31 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:51:53 2018 +0800
--
.../hadoop/util/CarbonInputFormatUtil.java | 20 +++
.../spark/rdd/NewCarbonDataLoadRDD.scala| 21 ++--
.../org/apache/spark/sql/CarbonSession.scala| 5 ++---
3 files changed, 24 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6944dd42/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
--
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 514428b..056c27b 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -22,6 +22,8 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Locale;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -39,6 +41,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
@@ -159,4 +162,21 @@ public class CarbonInputFormatUtil {
String jobtrackerID = createJobTrackerID(date);
return new JobID(jobtrackerID, batch);
}
+
+ public static void setS3Configurations(Configuration hadoopConf) {
+FileFactory.getConfiguration()
+.set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6944dd42/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 917fc88..e17824f 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,10 +41,10 @@ import
org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties,
CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.loading.{DataLoadExecutor,
FailureCauses, T
[40/50] [abbrv] carbondata git commit: [CARBONDATA-1997] Add CarbonWriter SDK API
[CARBONDATA-1997] Add CarbonWriter SDK API
Added a new module called store-sdk, and added a CarbonWriter API, it can be
used to write Carbondata files to a specified folder, without Spark and Hadoop
dependency. User can use this API in any environment.
This closes #1967
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bf4973c6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bf4973c6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bf4973c6
Branch: refs/heads/carbonstore-rebase5
Commit: bf4973c62fee3e51749b3a1a2d939b05581bb069
Parents: 653c51b
Author: Jacky Li
Authored: Sat Feb 10 19:44:23 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:36 2018 +0800
--
.../org/apache/carbondata/common/Strings.java | 40
.../apache/carbondata/common/StringsSuite.java | 53 +
.../core/metadata/schema/table/CarbonTable.java | 7 +
.../schema/table/CarbonTableBuilder.java| 72 +++
.../core/metadata/schema/table/TableSchema.java | 7 +
.../schema/table/TableSchemaBuilder.java| 107 ++
.../schema/table/CarbonTableBuilderSuite.java | 86
.../metadata/schema/table/CarbonTableTest.java | 12 +-
.../schema/table/TableSchemaBuilderSuite.java | 56 ++
.../carbondata/spark/util/DataLoadingUtil.scala | 45 +
pom.xml | 7 +
store/sdk/pom.xml | 130 +
.../carbondata/sdk/file/CSVCarbonWriter.java| 89 +
.../carbondata/sdk/file/CarbonWriter.java | 51 +
.../sdk/file/CarbonWriterBuilder.java | 194 +++
.../org/apache/carbondata/sdk/file/Field.java | 74 +++
.../org/apache/carbondata/sdk/file/Schema.java | 74 +++
.../sdk/file/CSVCarbonWriterSuite.java | 127
18 files changed, 1225 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf4973c6/common/src/main/java/org/apache/carbondata/common/Strings.java
--
diff --git a/common/src/main/java/org/apache/carbondata/common/Strings.java
b/common/src/main/java/org/apache/carbondata/common/Strings.java
new file mode 100644
index 000..23288dd
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/Strings.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common;
+
+import java.util.Objects;
+
+public class Strings {
+
+ /**
+ * Provide same function as mkString in Scala.
+ * This is added to avoid JDK 8 dependency.
+ */
+ public static String mkString(String[] strings, String delimeter) {
+Objects.requireNonNull(strings);
+Objects.requireNonNull(delimeter);
+StringBuilder builder = new StringBuilder();
+for (int i = 0; i < strings.length; i++) {
+ builder.append(strings[i]);
+ if (i != strings.length - 1) {
+builder.append(delimeter);
+ }
+}
+return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf4973c6/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
--
diff --git
a/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
b/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
new file mode 100644
index 000..65da32b
--- /dev/null
+++ b/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org
[41/50] [abbrv] carbondata git commit: [CARBONDATA-2156] Add interface annotation
[CARBONDATA-2156] Add interface annotation
InterfaceAudience and InterfaceStability annotation should be added for user
and developer
1.InetfaceAudience can be User and Developer
2.InterfaceStability can be Stable, Evolving, Unstable
This closes #1968
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/653c51b1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/653c51b1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/653c51b1
Branch: refs/heads/carbonstore-rebase5
Commit: 653c51b1b64e6d55f9483bbca7e1e85869de665e
Parents: 78aa2cc
Author: Jacky Li
Authored: Sun Feb 11 10:12:10 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:36 2018 +0800
--
.../common/annotations/InterfaceAudience.java | 58
.../common/annotations/InterfaceStability.java | 69
2 files changed, 127 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/653c51b1/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
new file mode 100644
index 000..fa9729d
--- /dev/null
+++
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * This annotation is ported and modified from Apache Hadoop project.
+ *
+ * Annotation to inform users of a package, class or method's intended
audience.
+ * Currently the audience can be {@link User}, {@link Developer}
+ *
+ * Public classes that are not marked with this annotation must be
+ * considered by default as {@link Developer}.
+ *
+ * External applications must only use classes that are marked {@link User}.
+ *
+ * Methods may have a different annotation that it is more restrictive
+ * compared to the audience classification of the class. Example: A class
+ * might be {@link User}, but a method may be {@link Developer}
+ */
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class InterfaceAudience {
+ /**
+ * Intended for use by any project or application.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface User { }
+
+ /**
+ * Intended only for developers to extend interface for CarbonData project
+ * For example, new Datamap implementations.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Developer { }
+
+ private InterfaceAudience() { } // Audience can't exist on its own
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/653c51b1/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
new file mode 100644
index 000..b8e5e52
--- /dev/null
+++
b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, softw
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 2a69f0d..a4d3d2b 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -272,7 +272,7 @@ public class CarbonCompactionUtil {
public static CarbonTable getNextTableToCompact(CarbonTable[] carbonTables,
List skipList) {
for (CarbonTable ctable : carbonTables) {
- String metadataPath = ctable.getMetaDataFilepath();
+ String metadataPath = ctable.getMetadataPath();
// check for the compaction required file and at the same time exclude
the tables which are
// present in the skip list.
if (CarbonCompactionUtil.isCompactionRequiredForTable(metadataPath) &&
!skipList
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index c141636..89326a3 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -169,15 +169,13 @@ public final class CarbonDataMergerUtil {
// End Timestamp.
// Table Update Status Metadata Update.
-AbsoluteTableIdentifier absoluteTableIdentifier =
+AbsoluteTableIdentifier identifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
-CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
-
SegmentUpdateStatusManager segmentUpdateStatusManager =
-new SegmentUpdateStatusManager(absoluteTableIdentifier);
+new SegmentUpdateStatusManager(identifier);
-SegmentStatusManager segmentStatusManager = new
SegmentStatusManager(absoluteTableIdentifier);
+SegmentStatusManager segmentStatusManager = new
SegmentStatusManager(identifier);
ICarbonLock updateLock =
segmentUpdateStatusManager.getTableUpdateStatusLock();
ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
@@ -224,7 +222,7 @@ public final class CarbonDataMergerUtil {
}
LoadMetadataDetails[] loadDetails =
- segmentStatusManager.readLoadMetadata(metaDataFilepath);
+ SegmentStatusManager.readLoadMetadata(metaDataFilepath);
for (LoadMetadataDetails loadDetail : loadDetails) {
if (loadsToMerge.contains(loadDetail)) {
@@ -237,18 +235,18 @@ public final class CarbonDataMergerUtil {
}
}
- segmentUpdateStatusManager
- .writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
- segmentStatusManager
-
.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(),
loadDetails);
+ segmentUpdateStatusManager.writeLoadDetailsIntoFile(
+ Arrays.asList(updateLists), timestamp);
+ SegmentStatusManager.writeLoadDetailsIntoFile(
+
CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadDetails);
status = true;
} else {
LOGGER.error("Not able to acquire the lock.");
status = false;
}
} catch (IOException e) {
-LOGGER.error("Error while updating metadata. The metadata file path is
" + carbonTablePath
-.getMetadataDirectoryPath());
+LOGGER.error("Error while updating metadata. The metadata file path is
" +
+CarbonTablePath.getMetadataPath(identifier.getTablePath()));
status = false;
} finally {
@@ -284,9 +282,9 @@ public final class CarbonDataMergerUtil {
String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel
carbonLoadModel,
CompactionType compactionType, String segmentFile) throws IOException {
boolean tableStatusUpdationStatus = false;
-AbsoluteTableIdentifier absoluteTableIdentifier =
+AbsoluteTableIdentifier identifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
-SegmentStatusManager segmentStatusManager = new
SegmentStatusManager(absoluteTableIdentifier);
+SegmentStatusManager segmentStatusManager = new
SegmentStatusManager(id
[32/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/78aa2cc3/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 527452a..11b3d43 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,14 +31,15 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import
org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import
org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -62,15 +63,21 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
* entry count
*/
private int entryCount;
+
/**
* return row
*/
- private IntermediateSortTempRow returnRow;
+ private Object[] returnRow;
+ private int dimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private DataType[] measureDataTypes;
private int readBufferSize;
private String compressorName;
- private IntermediateSortTempRow[] currentBuffer;
+ private Object[][] currentBuffer;
- private IntermediateSortTempRow[] backupBuffer;
+ private Object[][] backupBuffer;
private boolean isBackupFilled;
@@ -93,21 +100,27 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
private int numberOfObjectRead;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
- private Comparator comparator;
+ private int nullSetWordsLength;
+
+ private Comparator comparator;
+
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters
parameters) {
// set temp file
this.tempFile = tempFile;
+this.dimCnt = parameters.getDimColCount();
+this.complexCnt = parameters.getComplexDimColCount();
+this.measureCnt = parameters.getMeasureColCount();
+this.isNoDictionaryDimensionColumn =
parameters.getNoDictionaryDimnesionColumn();
+this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-this.tableFieldStat = new TableFieldStat(parameters);
-this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+
this.executorService = Executors.newFixedThreadPool(1);
-comparator = new
IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
+this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
+comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -156,17 +169,11 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
- @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- try {
-this.returnRow =
sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
-this.numberOfObjectRead++;
- } catch (IOException e) {
-throw new CarbonSortKeyAndGroupByException("Problems while reading
row", e);
- }
+ this.returnRow = getRowFromStream();
}
}
@@ -200,22 +207,63 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
}
/**
- * get a
[13/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index de97e82..540607d 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -58,7 +58,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,20 +68,20 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List dimColEvaluatorInfoList;
- protected List msrColEvalutorInfoList;
+ List dimColEvaluatorInfoList;
+ List msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
protected SegmentProperties segmentProperties;
/**
* it has index at which given dimension is stored in file
*/
- protected int[] dimensionBlocksIndex;
+ int[] dimensionChunkIndex;
/**
* it has index at which given measure is stored in file
*/
- protected int[] measureBlocksIndex;
+ int[] measureChunkIndex;
private Map complexDimensionInfoMap;
@@ -89,18 +89,18 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
* flag to check whether the filter dimension is present in current block
list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean[] isDimensionPresentInCurrentBlock;
+ boolean[] isDimensionPresentInCurrentBlock;
/**
* flag to check whether the filter measure is present in current block list
of measures.
* Applicable for restructure scenarios
*/
- protected boolean[] isMeasurePresentInCurrentBlock;
+ boolean[] isMeasurePresentInCurrentBlock;
/**
* is dimension column data is natural sorted
*/
- protected boolean isNaturalSorted;
+ boolean isNaturalSorted;
/**
* date direct dictionary generator
@@ -124,10 +124,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new
boolean[dimColEvaluatorInfoList.size()];
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
- this.dimensionBlocksIndex = new int[]{0};
+ this.dimensionChunkIndex = new int[]{0};
}
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new
ArrayList(20);
@@ -136,10 +136,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new
boolean[msrColEvalutorInfoList.size()];
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
- this.measureBlocksIndex = new int[] {0};
+ this.measureChunkInde
[01/50] [abbrv] carbondata git commit: [CARBONDATA-2211] in case of DDL HandOff should not be execute in thread [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/carbonstore-rebase5 8b94788a5 -> 7540cc9ca (forced update)
[CARBONDATA-2211] in case of DDL HandOff should not be execute in thread
1. DDL handoff will be executed in the blocking thread.
2. Auto handoff will be executed in a new non-blocking thread.
This closes #2008
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7f7ea4d7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7f7ea4d7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7f7ea4d7
Branch: refs/heads/carbonstore-rebase5
Commit: 7f7ea4d757d99b681512cb5f1369187f69f905c8
Parents: a0fc0be
Author: rahulforallp
Authored: Tue Feb 27 21:50:20 2018 +0530
Committer: QiangCai
Committed: Fri Mar 2 09:40:20 2018 +0800
--
.../CarbonAlterTableCompactionCommand.scala| 2 +-
.../carbondata/streaming/StreamHandoffRDD.scala| 17 +++--
.../streaming/CarbonAppendableStreamSink.scala | 3 ++-
3 files changed, 14 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7f7ea4d7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index f6019e4..9b9ca0e 100644
---
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -181,7 +181,7 @@ case class CarbonAlterTableCompactionCommand(
if (compactionType == CompactionType.STREAMING) {
StreamHandoffRDD.startStreamingHandoffThread(
carbonLoadModel,
-sqlContext.sparkSession)
+sqlContext.sparkSession, true)
return
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7f7ea4d7/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
--
diff --git
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index b03ee1e..a46ced5 100644
---
a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -279,15 +279,20 @@ object StreamHandoffRDD {
*/
def startStreamingHandoffThread(
carbonLoadModel: CarbonLoadModel,
- sparkSession: SparkSession
+ sparkSession: SparkSession,
+ isDDL: Boolean
): Unit = {
-// start a new thread to execute streaming segment handoff
-val handoffThread = new Thread() {
- override def run(): Unit = {
-iterateStreamingHandoff(carbonLoadModel, sparkSession)
+if (isDDL) {
+ iterateStreamingHandoff(carbonLoadModel, sparkSession)
+} else {
+ // start a new thread to execute streaming segment handoff
+ val handoffThread = new Thread() {
+override def run(): Unit = {
+ iterateStreamingHandoff(carbonLoadModel, sparkSession)
+}
}
+ handoffThread.start()
}
-handoffThread.start()
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7f7ea4d7/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
--
diff --git
a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index f2f9853..312d24e 100644
---
a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++
b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -178,7 +178,8 @@ class CarbonAppendableStreamSink(
if (enableAutoHandoff) {
StreamHandoffRDD.startStreamingHandoffThread(
carbonLoadModel,
- sparkSession)
+ sparkSession,
+ false)
}
}
}
[26/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after rebasing master
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/65daaca7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/65daaca7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/65daaca7
Branch: refs/heads/carbonstore-rebase5
Commit: 65daaca7de643f79e807a6d815a4cc6205ef2dad
Parents: d115c47
Author: Jacky Li
Authored: Fri Feb 9 01:39:20 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:35 2018 +0800
--
.../scan/filter/FilterExpressionProcessor.java | 2 +-
.../filter/executer/FalseFilterExecutor.java| 16 +++---
.../FalseConditionalResolverImpl.java | 4 ++--
.../apache/carbondata/core/util/CarbonUtil.java | 1 -
.../core/util/path/CarbonTablePath.java | 6 +++---
.../spark/rdd/AggregateDataMapCompactor.scala | 12 +--
.../preaaggregate/PreAggregateListeners.scala | 22 +---
.../CarbonAlterTableDataTypeChangeCommand.scala | 17 ---
.../schema/CarbonAlterTableRenameCommand.scala | 3 ++-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 12 ++-
.../processing/util/CarbonLoaderUtil.java | 5 +++--
11 files changed, 51 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65daaca7/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index b882b51..26b202f 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -398,7 +398,7 @@ public class FilterExpressionProcessor implements
FilterProcessor {
ConditionalExpression condExpression = null;
switch (filterExpressionType) {
case FALSE:
-return new FalseConditionalResolverImpl(expression, false, false,
tableIdentifier);
+return new FalseConditionalResolverImpl(expression, false, false);
case TRUE:
return new TrueConditionalResolverImpl(expression, false, false);
case EQUALS:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/65daaca7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
index 2d2a15c..75a6ec3 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import
org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
/**
@@ -33,8 +33,8 @@ import org.apache.carbondata.core.util.BitSetGroup;
public class FalseFilterExecutor implements FilterExecuter {
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean
useBitsetPipeline)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks blocksChunkHolder,
+ boolean useBitsetPipeline) throws FilterUnsupportedException,
IOException {
int numberOfPages = blocksChunkHolder.getDataBlock().numberOfPages();
BitSetGroup group = new BitSetGroup(numberOfPages);
for (int i = 0; i < numberOfPages; i++) {
@@ -44,17 +44,19 @@ public class FalseFilterExecutor implements FilterExecuter {
return group;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][]
blockMinValue) {
-
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue)
{
return new BitSet();
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws
IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks block
[31/50] [abbrv] carbondata git commit: [REBASE] resolve conflict after rebasing to master
[REBASE] resolve conflict after rebasing to master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/90629314
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/90629314
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/90629314
Branch: refs/heads/carbonstore-rebase5
Commit: 90629314839bf639fda95c3d267983de1012cb05
Parents: b5908d2
Author: Jacky Li
Authored: Tue Feb 27 08:51:25 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:35 2018 +0800
--
.../carbondata/core/datamap/TableDataMap.java | 2 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 5 ++--
.../core/datamap/dev/DataMapFactory.java| 2 +-
.../core/indexstore/BlockletDetailsFetcher.java | 2 +-
.../indexstore/SegmentPropertiesFetcher.java| 3 +-
.../blockletindex/BlockletDataMap.java | 2 +-
.../blockletindex/BlockletDataMapFactory.java | 21 ++---
.../core/metadata/SegmentFileStore.java | 2 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 1 +
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 1 +
...velRangeLessThanEqualFilterExecuterImpl.java | 1 +
.../RowLevelRangeLessThanFiterExecuterImpl.java | 1 +
.../SegmentUpdateStatusManager.java | 26
.../apache/carbondata/core/util/CarbonUtil.java | 16 --
.../testsuite/datamap/CGDataMapTestCase.scala | 26
.../testsuite/datamap/DataMapWriterSuite.scala | 19 ++--
.../testsuite/datamap/FGDataMapTestCase.scala | 31 +---
.../iud/DeleteCarbonTableTestCase.scala | 2 +-
.../TestInsertAndOtherCommandConcurrent.scala | 14 +
.../StandardPartitionTableCleanTestCase.scala | 12
.../spark/rdd/NewCarbonDataLoadRDD.scala| 2 +-
.../carbondata/spark/util/DataLoadingUtil.scala | 4 +--
.../CreatePreAggregateTableCommand.scala| 2 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 3 +-
.../datamap/DataMapWriterListener.java | 2 +-
.../loading/model/CarbonLoadModel.java | 2 +-
.../processing/merger/CarbonDataMergerUtil.java | 15 +++---
.../merger/CompactionResultSortProcessor.java | 6 ++--
.../merger/RowResultMergerProcessor.java| 6 ++--
.../partition/spliter/RowResultProcessor.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 4 +--
.../processing/util/CarbonLoaderUtil.java | 2 +-
32 files changed, 104 insertions(+), 136 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/90629314/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index eed650e3..2a6ceaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -143,7 +143,7 @@ public final class TableDataMap extends
OperationEventListener {
blocklets.addAll(
dataMap.prune(
filterExp,
-
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()),
+
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment()),
partitions));
}
BlockletSerializer serializer = new BlockletSerializer();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/90629314/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
index bcc9bad..de6dcb1 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -35,10 +36,10 @@ public abstract class AbstractDataMapWriter {
protected String writeDirectoryPath;
- public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String
segmentId,
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, Segment
segment,
String writeDirectoryPath) {
this.identifier = identi
[05/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove partitionId in CarbonTablePath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2e28c156/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index d321405..58009af 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -65,21 +66,21 @@ public class DataWriterProcessorStepImpl extends
AbstractDataLoadProcessorStep {
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier,
String partitionId) {
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
String[] storeLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
-tableIdentifier.getTableName(),
String.valueOf(configuration.getTaskNo()), partitionId,
-configuration.getSegmentId() + "", false, false);
+tableIdentifier.getTableName(),
String.valueOf(configuration.getTaskNo()),
+configuration.getSegmentId(), false, false);
CarbonDataProcessorUtil.createLocations(storeLocation);
return storeLocation;
}
- public CarbonFactDataHandlerModel getDataHandlerModel(int partitionId) {
+ public CarbonFactDataHandlerModel getDataHandlerModel() {
CarbonTableIdentifier tableIdentifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
-String[] storeLocation = getStoreLocation(tableIdentifier,
String.valueOf(partitionId));
+String[] storeLocation = getStoreLocation(tableIdentifier);
return
CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration,
-storeLocation, partitionId, 0);
+storeLocation, 0, 0);
}
@Override public Iterator[] execute() throws
CarbonDataLoadingException {
@@ -89,11 +90,11 @@ public class DataWriterProcessorStepImpl extends
AbstractDataLoadProcessorStep {
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-
.recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+
.recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
for (Iterator iterator : iterators) {
-String[] storeLocation = getStoreLocation(tableIdentifier,
String.valueOf(i));
+String[] storeLocation = getStoreLocation(tableIdentifier);
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
.createCarbonFactDataHandlerModel(configuration, storeLocation, i,
0);
@@ -147,10 +148,11 @@ public class DataWriterProcessorStepImpl extends
AbstractDataLoadProcessorStep {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-.recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+
.recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-.recordMdkGenerateTotalTime(configuration.getPartitionId(),
System.currentTimeMillis());
+.recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) throws
CarbonDataLoadingException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2e28c156/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 3f1430d..c141636 100644
---
a/processing/src/main/
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index a1ccab3..4293536 100644
---
a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++
b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -39,21 +39,19 @@ public class CarbonFormatDirectoryStructureTest {
*/
@Test public void testTablePathStructure() throws IOException {
CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("d1",
"t1", UUID.randomUUID().toString());
-CarbonStorePath carbonStorePath = new CarbonStorePath(CARBON_STORE);
-AbsoluteTableIdentifier absoluteTableIdentifier =
+AbsoluteTableIdentifier identifier =
AbsoluteTableIdentifier.from(CARBON_STORE + "/d1/t1", tableIdentifier);
-CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
-assertTrue(carbonTablePath.getPath().replace("\\",
"/").equals(CARBON_STORE + "/d1/t1"));
-assertTrue(carbonTablePath.getSchemaFilePath().replace("\\",
"/").equals(CARBON_STORE + "/d1/t1/Metadata/schema"));
-assertTrue(carbonTablePath.getTableStatusFilePath().replace("\\", "/")
+assertTrue(identifier.getTablePath().replace("\\",
"/").equals(CARBON_STORE + "/d1/t1"));
+
assertTrue(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()).replace("\\",
"/").equals(CARBON_STORE + "/d1/t1/Metadata/schema"));
+
assertTrue(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()).replace("\\",
"/")
.equals(CARBON_STORE + "/d1/t1/Metadata/tablestatus"));
-assertTrue(carbonTablePath.getDictionaryFilePath("t1_c1").replace("\\",
"/")
+
assertTrue(CarbonTablePath.getDictionaryFilePath(identifier.getTablePath(),
"t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dict"));
-
assertTrue(carbonTablePath.getDictionaryMetaFilePath("t1_c1").replace("\\", "/")
+
assertTrue(CarbonTablePath.getDictionaryMetaFilePath(identifier.getTablePath(),
"t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
-assertTrue(carbonTablePath.getSortIndexFilePath("t1_c1").replace("\\", "/")
+
assertTrue(CarbonTablePath.getSortIndexFilePath(identifier.getTablePath(),"t1_c1").replace("\\",
"/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
-assertTrue(carbonTablePath.getCarbonDataFilePath("2", 3, 4L, 0, 0,
"999").replace("\\", "/")
+
assertTrue(CarbonTablePath.getCarbonDataFilePath(identifier.getTablePath(),
"2", 3, 4L, 0, 0, "999").replace("\\", "/")
.equals(CARBON_STORE +
"/d1/t1/Fact/Part0/Segment_2/part-3-4_batchno0-0-999.carbondata"));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9daad358/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
deleted file mode 100644
index 91384c1..000
---
a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.util.path;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertTrue;
-
-/**
- * test shared dictionary paths
- */
-public class CarbonFormatSharedDictionaryTest {
-
- private final String CARBON_STORE = "/opt/carbonstore";
-
- /**
- * test shared dictionary location
- */
- @Test public void testSharedDimentionLocation() throws IOException {
-as
[39/50] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fbe97786
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fbe97786
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fbe97786
Branch: refs/heads/carbonstore-rebase5
Commit: fbe97786a252a4502ac10e891292e35a08a2c11f
Parents: f9d46b1
Author: xuchuanyin
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 15:52:36 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/block/TableBlockInfo.java| 29 ++
.../carbondata/core/util/CarbonProperties.java | 11 +
docs/useful-tips-on-carbondata.md | 1 +
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 ++
.../processing/util/CarbonLoaderUtil.java | 480 ---
.../processing/util/CarbonLoaderUtilTest.java | 125 +
10 files changed, 545 insertions(+), 175 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fbe97786/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index bcfeba0..a6bf60f 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,4 +114,14 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
+ /**
+ * enable block size based block allocation while loading data. By default,
carbondata assigns
+ * blocks to node based on block number. If this option is set to `true`,
carbondata will
+ * consider block size first and make sure that all the nodes will process
almost equal size of
+ * data. This option is especially useful when you encounter skewed data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
+ = "carbon.load.skewedDataOptimization.enabled";
+ public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fbe97786/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a7bfdba..c0cebe0 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -98,6 +100,20 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
+ /**
+ * comparator to sort by block size in descending order.
+ * Since each line is not exactly the same, the size of a InputSplit may
differs,
+ * so we allow some deviation for these splits.
+ */
+ public static final Comparator DATA_SIZE_DESC_COMPARATOR =
+ new Comparator() {
+@Override public int compare(Distributable o1, Distributable o2) {
+ long diff =
+ ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
+ return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
+}
+ };
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -434,4 +450,17 @@ public class TableBlockInfo implements Distributable,
Serializable {
[28/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d0990532/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
--
diff --git
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 78544d3..fe0bbcf 100644
---
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.datamap.examples;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
@@ -29,17 +28,18 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
-public class MinMaxDataWriter implements DataMapWriter {
+public class MinMaxDataWriter extends AbstractDataMapWriter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
@@ -50,17 +50,23 @@ public class MinMaxDataWriter implements DataMapWriter {
private Map blockMinMaxMap;
- private String blockPath;
+ private String dataWritePath;
+ public MinMaxDataWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
+super(identifier, segmentId, dataWritePath);
+this.identifier = identifier;
+this.segmentId = segmentId;
+this.dataWritePath = dataWritePath;
+ }
- @Override public void onBlockStart(String blockId, String blockPath) {
+ @Override public void onBlockStart(String blockId) {
pageLevelMax = null;
pageLevelMin = null;
blockletLevelMax = null;
blockletLevelMin = null;
blockMinMaxMap = null;
blockMinMaxMap = new HashMap();
-this.blockPath = blockPath;
}
@Override public void onBlockEnd(String blockId) {
@@ -161,7 +167,7 @@ public class MinMaxDataWriter implements DataMapWriter {
List tempMinMaxIndexBlockDetails = null;
tempMinMaxIndexBlockDetails = loadBlockDetails();
try {
- writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockId);
} catch (IOException ex) {
LOGGER.info(" Unable to write the file");
}
@@ -178,7 +184,6 @@ public class MinMaxDataWriter implements DataMapWriter {
tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
tmpminMaxIndexBlockDetails.setBlockletId(index);
- tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
}
return minMaxIndexBlockDetails;
@@ -187,22 +192,19 @@ public class MinMaxDataWriter implements DataMapWriter {
/**
* Write the data to a file. This is JSON format file.
* @param minMaxIndexBlockDetails
- * @param blockPath
* @param blockId
* @throws IOException
*/
public void writeMinMaxIndexFile(List
minMaxIndexBlockDetails,
- String blockPath, String blockId) throws IOException {
-String filePath = blockPath.substring(0,
blockPath.lastIndexOf(File.separator) + 1) + blockId
-+ ".minmaxindex";
+ String blockId) throws IOException {
+String filePath = dataWritePath +"/" + blockId + ".minmaxindex";
BufferedWriter brWriter = null;
DataOutputStream dataOutStream = null;
try {
FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
dataOutStream = FileFactory.getDataOutputStream(filePath,
FileFactory.getFileType(filePath));
Gson gsonObjectToWrite = new Gson();
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
"UTF-8"));
String minmaxIndexData =
g
[1/2] carbondata git commit: [REBASE] Solve conflict after merging master [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/carbonstore-rebase5 7540cc9ca -> c305f309e (forced update)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index d2faef5..142b2cb 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -21,7 +21,16 @@ import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -32,7 +41,6 @@ import
org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
@@ -592,10 +600,6 @@ public final class CarbonDataMergerUtil {
List segmentsToBeMerged =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-CarbonTable carbonTable =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
-CarbonTableIdentifier tableIdentifier =
carbonTable.getCarbonTableIdentifier();
-
-
// total length
long totalLength = 0;
@@ -1013,7 +1017,8 @@ public final class CarbonDataMergerUtil {
CarbonFile[] updateDeltaFiles = null;
Set uniqueBlocks = new HashSet();
-String segmentPath =
CarbonTablePath.getSegmentPath(absoluteTableIdentifier.getTablePath(),
seg.getSegmentNo());
+String segmentPath = CarbonTablePath.getSegmentPath(
+absoluteTableIdentifier.getTablePath(), seg.getSegmentNo());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath,
FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ea11e22..ebcf944 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -405,8 +405,8 @@ public class CompactionResultSortProcessor extends
AbstractResultProcessor {
partitionSpec.getLocation().toString() +
CarbonCommonConstants.FILE_SEPARATOR
+ carbonLoadModel.getFactTimeStamp() + ".tmp";
} else {
- carbonStoreLocation = CarbonDataProcessorUtil
- .createCarbonStoreLocation(carbonLoadModel.getDatabaseName(),
tableName, carbonLoadModel.getSegmentId());
+ carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation(
+ carbonLoadModel.getDatabaseName(), tableName,
carbonLoadModel.getSegmentId());
}
CarbonFactDataHandlerModel carbonFactDataHandlerModel =
CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable,
segmentProperties, tableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index 278d5bb..2616def 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@
[2/2] carbondata git commit: [REBASE] Solve conflict after merging master
[REBASE] Solve conflict after merging master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c305f309
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c305f309
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c305f309
Branch: refs/heads/carbonstore-rebase5
Commit: c305f309e4556bfc98f01e0dff41b8411c992a12
Parents: 8a9dd8b
Author: Jacky Li
Authored: Tue Feb 27 11:26:30 2018 +0800
Committer: Jacky Li
Committed: Fri Mar 2 17:48:31 2018 +0800
--
.../carbondata/core/datamap/dev/DataMap.java| 9 +-
.../core/datamap/dev/DataMapFactory.java| 2 +-
.../exception/ConcurrentOperationException.java | 16 +-
.../core/indexstore/BlockletDetailsFetcher.java | 3 +-
.../blockletindex/BlockletDataMap.java | 3 +-
.../blockletindex/SegmentIndexFileStore.java| 2 -
.../core/metadata/PartitionMapFileStore.java| 0
.../scan/executor/util/RestructureUtil.java | 6 +-
.../statusmanager/SegmentStatusManager.java | 10 +-
.../SegmentUpdateStatusManager.java | 7 +-
datamap/examples/pom.xml| 145 +++--
.../datamap/examples/MinMaxDataMap.java | 3 +-
.../datamap/examples/MinMaxDataMapFactory.java | 26 +--
.../datamap/examples/MinMaxDataWriter.java | 8 +-
.../CarbonStreamSparkStreamingExample.scala | 14 +-
...CarbonStructuredStreamingWithRowParser.scala | 8 +-
.../hadoop/api/CarbonTableInputFormat.java | 5 +-
.../preaggregate/TestPreAggCreateCommand.scala | 2 +-
.../TestInsertAndOtherCommandConcurrent.scala | 2 +-
.../StandardPartitionGlobalSortTestCase.scala | 2 +-
.../exception/ProcessMetaDataException.java | 2 +
.../org/apache/carbondata/api/CarbonStore.scala | 6 +-
.../carbondata/spark/load/CsvRDDHelper.scala| 157 +++
.../load/DataLoadProcessBuilderOnSpark.scala| 3 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 2 -
.../command/carbonTableSchemaCommon.scala | 6 +-
.../CarbonAlterTableCompactionCommand.scala | 3 +-
.../management/CarbonCleanFilesCommand.scala| 2 +-
.../CarbonDeleteLoadByIdCommand.scala | 2 +-
.../CarbonDeleteLoadByLoadDateCommand.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 28 ++--
.../CarbonProjectForDeleteCommand.scala | 2 +-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 3 -
.../TestStreamingTableWithRowParser.scala | 9 +-
.../vectorreader/AddColumnTestCases.scala | 1 +
.../datamap/DataMapWriterListener.java | 3 +-
.../loading/model/CarbonLoadModelBuilder.java | 34 +++-
.../processing/loading/model/LoadOption.java| 15 +-
.../processing/merger/CarbonDataMergerUtil.java | 19 ++-
.../merger/CompactionResultSortProcessor.java | 4 +-
.../merger/RowResultMergerProcessor.java| 4 +-
.../partition/spliter/RowResultProcessor.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 3 +-
store/sdk/pom.xml | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java| 8 +-
49 files changed, 384 insertions(+), 221 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 4a68286..fdeacff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -38,18 +38,13 @@ public interface DataMap {
/**
* Prune the datamap with filter expression and partition information. It
returns the list of
* blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
*/
- List prune(FilterResolverIntf filterExp, SegmentProperties
segmentProperties, List partitions);
+ List prune(FilterResolverIntf filterExp, SegmentProperties
segmentProperties,
+ List partitions);
// TODO Move this method to Abstract class
/**
* Validate whether the current segment needs to be fetching the required
data
- *
- * @param filterExp
- * @return
*/
boolean isScanRequired(FilterResolverIntf filterExp);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/core/src/main/java/org/apache/carbondata/core/datamap/dev/
carbondata git commit: [CARBONDATA-2098] Optimize pre-aggregate documentation
Repository: carbondata Updated Branches: refs/heads/master 74f5d67c0 -> 3e36639ed [CARBONDATA-2098] Optimize pre-aggregate documentation optimize pre-aggregate documentation move to separate file add more examples This closes #2022 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3e36639e Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3e36639e Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3e36639e Branch: refs/heads/master Commit: 3e36639eda45933a036ea364dba1088c5f4e57ec Parents: 74f5d67 Author: sraghunandan Authored: Fri Mar 2 17:02:39 2018 +0530 Committer: Jacky Li Committed: Sat Mar 3 09:44:02 2018 +0800 -- docs/data-management-on-carbondata.md | 242 -- docs/preaggregate-guide.md | 313 +++ .../examples/PreAggregateTableExample.scala | 50 ++- 3 files changed, 362 insertions(+), 243 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/3e36639e/docs/data-management-on-carbondata.md -- diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md index ea80d41..2aa4a49 100644 --- a/docs/data-management-on-carbondata.md +++ b/docs/data-management-on-carbondata.md @@ -26,7 +26,6 @@ This tutorial is going to introduce all commands and data operations on CarbonDa * [UPDATE AND DELETE](#update-and-delete) * [COMPACTION](#compaction) * [PARTITION](#partition) -* [PRE-AGGREGATE TABLES](#pre-aggregate-tables) * [BUCKETING](#bucketing) * [SEGMENT MANAGEMENT](#segment-management) @@ -859,247 +858,6 @@ This tutorial is going to introduce all commands and data operations on CarbonDa * The partitioned column can be excluded from SORT_COLUMNS, this will let other columns to do the efficient sorting. * When writing SQL on a partition table, try to use filters on the partition column. - -## PRE-AGGREGATE TABLES - CarbonData supports pre aggregating of data so that OLAP kind of queries can fetch data - much faster. Aggregate tables are created as datamaps so that the handling is as efficient as - other indexing support. Users can create as many aggregate tables they require as datamaps to - improve their query performance, provided the storage requirements and loading speeds are - acceptable. - - For main table called **sales** which is defined as - - ``` - CREATE TABLE sales ( -order_time timestamp, -user_id STRING, -sex STRING, -country STRING, -quantity INT, -price BIGINT) - STORED BY 'carbondata' - ``` - - user can create pre-aggregate tables using the DDL - - ``` - CREATE DATAMAP agg_sales - ON TABLE sales - USING "preaggregate" - AS - SELECT country, sex, sum(quantity), avg(price) - FROM sales - GROUP BY country, sex - ``` - -Functions supported in pre-aggregate tables - -| Function | Rollup supported | -|---|| -| SUM | Yes | -| AVG | Yes | -| MAX | Yes | -| MIN | Yes | -| COUNT | Yes | - - -# How pre-aggregate tables are selected -For the main table **sales** and pre-aggregate table **agg_sales** created above, queries of the -kind -``` -SELECT country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex - -SELECT sex, sum(quantity) from sales GROUP BY sex - -SELECT sum(price), country from sales GROUP BY country -``` - -will be transformed by Query Planner to fetch data from pre-aggregate table **agg_sales** - -But queries of kind -``` -SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex - -SELECT sex, avg(quantity) from sales GROUP BY sex - -SELECT country, max(price) from sales GROUP BY country -``` - -will fetch the data from the main table **sales** - -# Loading data to pre-aggregate tables -For existing table with loaded data, data load to pre-aggregate table will be triggered by the -CREATE DATAMAP statement when user creates the pre-aggregate table. -For incremental loads after aggregates tables are created, loading data to main table triggers -the load to pre-aggregate tables once main table loading is complete. These loads are automic -meaning that data on main table and aggregate tables are only visible to the user after all tables -are loaded - -# Querying data from pre-aggregate tables -Pre-aggregate tables cannot be queries directly. Queries are to be made on main table. Internally -carbondata will check associated pre-aggregate tables with the main table, and if the -pre-aggregate tables satisfy the query condition, the plan is transformed automatically to use -pre-aggregate table to fetch the data. - -# Compacting pre-aggreg
carbondata git commit: [CARBONDATA-2204] Optimized number of reads of tablestatus file while querying
Repository: carbondata
Updated Branches:
refs/heads/master bfd77f69f -> c125f0caa
[CARBONDATA-2204] Optimized number of reads of tablestatus file while querying
This PR avoid reading status file multiple times. For first time query, it
reads 2 times(Needed for datamap refresher) and 1 time for second query onwards.
This closes #1999
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c125f0ca
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c125f0ca
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c125f0ca
Branch: refs/heads/master
Commit: c125f0caa58f1e7cfa7d10b52ab53364f1895c35
Parents: bfd77f6
Author: ravipesala
Authored: Mon Feb 26 18:36:03 2018 +0530
Committer: Jacky Li
Committed: Sat Mar 3 13:50:16 2018 +0800
--
.../core/constants/CarbonCommonConstants.java | 4 -
.../statusmanager/SegmentStatusManager.java | 122 +--
.../SegmentUpdateStatusManager.java | 28 +++--
.../core/util/path/CarbonTablePath.java | 9 +-
.../hadoop/api/CarbonTableInputFormat.java | 24 ++--
.../hadoop/test/util/StoreCreator.java | 8 +-
.../presto/util/CarbonDataStoreCreator.scala| 2 +-
.../carbondata/processing/StoreCreator.java | 4 +-
8 files changed, 108 insertions(+), 93 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c125f0ca/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index fa2b7d8..b2a3375 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -650,10 +650,6 @@ public final class CarbonCommonConstants {
*/
public static final int DEFAULT_MAX_QUERY_EXECUTION_TIME = 60;
/**
- * LOADMETADATA_FILENAME
- */
- public static final String LOADMETADATA_FILENAME = "tablestatus";
- /**
* TABLE UPDATE STATUS FILENAME
*/
public static final String TABLEUPDATESTATUS_FILENAME = "tableupdatestatus";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c125f0ca/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index 76c2dc7..ab849ce 100755
---
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -98,6 +98,15 @@ public class SegmentStatusManager {
* @throws IOException
*/
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments() throws
IOException {
+return getValidAndInvalidSegments(null);
+ }
+
+ /**
+ * get valid segment for given load status details.
+ *
+ */
+ public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments(
+ LoadMetadataDetails[] loadMetadataDetails) throws IOException {
// @TODO: move reading LoadStatus file to separate class
List listOfValidSegments = new ArrayList<>(10);
@@ -108,73 +117,56 @@ public class SegmentStatusManager {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
-String dataPath = carbonTablePath.getTableStatusFilePath();
-DataInputStream dataInputStream = null;
-// Use GSON to deserialize the load information
-Gson gson = new Gson();
-
-AtomicFileOperations fileOperation =
-new AtomicFileOperationsImpl(dataPath,
FileFactory.getFileType(dataPath));
-LoadMetadataDetails[] loadFolderDetailsArray;
try {
- if (FileFactory.isFileExist(dataPath,
FileFactory.getFileType(dataPath))) {
-dataInputStream = fileOperation.openForRead();
-BufferedReader buffReader =
-new BufferedReader(new InputStreamReader(dataInputStream,
"UTF-8"));
-loadFolderDetailsArray = gson.fromJson(buffReader,
LoadMetadataDetails[].class);
-// if loadFolderDetailsArray is null, assign a empty array
-if (null == loadFolderDetailsArray) {
- loadFolderDetailsArray = new LoadMetadataDetails[0];
-}
-//just directly iterate Array
-for (LoadMetadataDetails segment : loadFolderDetailsArray) {
- if (Segm
[47/50] [abbrv] carbondata git commit: Support generating assembling JAR for store-sdk module
Support generating assembling JAR for store-sdk module
Support generating assembling JAR for store-sdk module and remove junit
dependency
This closes #1976
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d32c0cf3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d32c0cf3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d32c0cf3
Branch: refs/heads/carbonstore-rebase5
Commit: d32c0cf306a331f2f7e23c57438d2c4acff52620
Parents: e83d910
Author: Jacky Li
Authored: Tue Feb 13 09:12:09 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:08:30 2018 +0800
--
common/pom.xml| 2 +
core/pom.xml | 2 +
hadoop/pom.xml| 1 +
integration/presto/pom.xml| 3 +-
integration/spark-common-cluster-test/pom.xml | 2 +-
integration/spark-common-test/pom.xml | 3 +-
integration/spark-common/pom.xml | 2 +-
integration/spark2/pom.xml| 2 +-
pom.xml | 5 +++
processing/pom.xml| 1 +
store/sdk/pom.xml | 50 +-
streaming/pom.xml | 1 -
12 files changed, 66 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 5550129..433d575 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,10 +42,12 @@
junit
junit
+ test
org.jmockit
jmockit
+ test
org.apache.hadoop
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 92c9607..824de0d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,10 +70,12 @@
org.jmockit
jmockit
+ test
junit
junit
+ test
org.apache.spark
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/hadoop/pom.xml
--
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 2aaac99..c3964c5 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -42,6 +42,7 @@
junit
junit
+ test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index aaaf175..0abcf38 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -193,7 +193,7 @@
org.scalatest
- scalatest_2.11
+ scalatest_${scala.binary.version}
org.apache.zookeeper
@@ -330,7 +330,6 @@
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/integration/spark-common-cluster-test/pom.xml
--
diff --git a/integration/spark-common-cluster-test/pom.xml
b/integration/spark-common-cluster-test/pom.xml
index fd907a3..028da11 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -49,11 +49,11 @@
junit
junit
+ test
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/integration/spark-common-test/pom.xml
--
diff --git a/integration/spark-common-test/pom.xml
b/integration/spark-common-test/pom.xml
index 67a2317..d1c04ae 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -106,16 +106,17 @@
junit
junit
+ test
org.scalatest
scalatest_${scala.binary.version}
- 2.2.1
test
org.jmockit
jmockit
+ test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d32c0cf3/integration/spark-common/pom.xml
--
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 295d62b..16f327d 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -58,11 +58,11 @@
junit
junit
+ test
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b9125b6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
--
diff --git
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 40b5cfc..753e637 100644
---
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -34,7 +34,7 @@ import
org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableRenamePostEvent,
AlterTableRenamePreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.SchemaEvolutionEntry
import org.apache.carbondata.spark.exception.{ConcurrentOperationException,
MalformedCarbonCommandException}
@@ -97,8 +97,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
val oldTableIdentifier = carbonTable.getAbsoluteTableIdentifier
DataMapStoreManager.getInstance().clearDataMaps(oldTableIdentifier)
// get the latest carbon table and check for column existence
- val oldTablePath = CarbonStorePath.getCarbonTablePath(oldTableIdentifier)
- val tableMetadataFile = oldTablePath.getPath
+ val tableMetadataFile = oldTableIdentifier.getTablePath
val operationContext = new OperationContext
// TODO: Pass new Table Path in pre-event.
val alterTableRenamePreEvent: AlterTableRenamePreEvent =
AlterTableRenamePreEvent(
@@ -108,7 +107,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
sparkSession)
OperationListenerBus.getInstance().fireEvent(alterTableRenamePreEvent,
operationContext)
val tableInfo: org.apache.carbondata.format.TableInfo =
-metastore.getThriftTableInfo(oldTablePath)(sparkSession)
+metastore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaEvolutionEntry = new
SchemaEvolutionEntry(System.currentTimeMillis)
schemaEvolutionEntry.setTableName(newTableName)
timeStamp = System.currentTimeMillis()
@@ -117,7 +116,8 @@ private[sql] case class CarbonAlterTableRenameCommand(
val fileType = FileFactory.getFileType(tableMetadataFile)
val newTableIdentifier = new CarbonTableIdentifier(oldDatabaseName,
newTableName, carbonTable.getCarbonTableIdentifier.getTableId)
- var newTablePath = CarbonUtil.getNewTablePath(oldTablePath,
newTableIdentifier.getTableName)
+ var newTablePath = CarbonTablePath.getNewTablePath(
+oldTableIdentifier.getTablePath, newTableIdentifier.getTableName)
metastore.removeTableFromMetadata(oldDatabaseName, oldTableName)
val hiveClient =
sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog]
.getClient()
@@ -139,9 +139,9 @@ private[sql] case class CarbonAlterTableRenameCommand(
// changed the rename order to deal with situation when carbon table and
hive table
// will point to the same tablePath
if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
-val rename = FileFactory.getCarbonFile(oldTablePath.getPath, fileType)
- .renameForce(oldTablePath.getParent.toString +
CarbonCommonConstants.FILE_SEPARATOR +
- newTableName)
+val rename =
FileFactory.getCarbonFile(oldTableIdentifier.getTablePath, fileType)
+ .renameForce(
+CarbonTablePath.getNewTablePath(oldTableIdentifier.getTablePath,
newTableName))
if (!rename) {
renameBadRecords(newTableName, oldTableName, oldDatabaseName)
sys.error(s"Folder rename failed for table
$oldDatabaseName.$oldTableName")
@@ -149,7 +149,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
}
val updatedParts = updatePartitionLocations(
partitions,
-oldTablePath.getPath,
+oldTableIdentifier.getTablePath,
newTablePath,
sparkSession)
@@ -191,13 +191,11 @@ private[sql] case class CarbonAlterTableRenameCommand(
case e: Exception =>
LOGGER.error(e, "Rename table failed: " + e.getMessage)
if (carbonTable != null) {
- AlterTableUtil
-.revertRenameTableChanges(oldTableIdentifier,
- newTableName,
- carbonTable.getTablePath,
- carbonTable.getCarbo
[10/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 447ab46..547ecaa 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,12 +35,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import
org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -73,7 +72,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends
RowLevelFilterExecut
comparator =
Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
ifDefaultValueMatchesFilter();
-if (isDimensionPresentInCurrentBlock[0] == true) {
+if (isDimensionPresentInCurrentBlock[0]) {
isNaturalSorted =
dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
&& dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
}
@@ -120,11 +119,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl
extends RowLevelFilterExecut
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] ||
isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
-minValue = blockMinValue[measureBlocksIndex[0] +
lastDimensionColOrdinal];
+minValue = blockMinValue[measureChunkIndex[0] +
lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType());
} else {
-minValue = blockMinValue[dimensionBlocksIndex[0]];
+minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -170,67 +169,69 @@ public class RowLevelRangeLessThanFiterExecuterImpl
extends RowLevelFilterExecut
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean
useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks
rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] &&
!isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
-
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+
.createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
-
segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-blockChunkHolder.getDimensionRawDataChunk()[blockIndex] =
blockChunkHolder.getDataBlock()
-.getDimensionC
[17/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
[CARBONDATA-2099] Refactor query scan process to improve readability Unified concepts in scan process flow: 1.QueryModel contains all parameter for scan, it is created by API in CarbonTable. (In future, CarbonTable will be the entry point for various table operations) 2.Use term ColumnChunk to represent one column in one blocklet, and use ChunkIndex in reader to read specified column chunk 3.Use term ColumnPage to represent one page in one ColumnChunk 4.QueryColumn => ProjectionColumn, indicating it is for projection This closes #1874 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/55c4e438 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/55c4e438 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/55c4e438 Branch: refs/heads/carbonstore-rebase5 Commit: 55c4e43828d3918fb8657ca7279a89a3cad4d667 Parents: 03157f9 Author: Jacky Li Authored: Tue Jan 30 21:24:04 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 18:15:06 2018 +0800 -- .../dictionary/AbstractDictionaryCache.java | 3 +- .../cache/dictionary/DictionaryCacheLoader.java | 7 +- .../dictionary/DictionaryCacheLoaderImpl.java | 11 +- .../core/datastore/BTreeBuilderInfo.java| 6 - .../carbondata/core/datastore/DataRefNode.java | 81 +-- .../carbondata/core/datastore/FileHolder.java | 118 .../carbondata/core/datastore/FileReader.java | 114 +++ .../core/datastore/block/SegmentProperties.java | 50 +- .../chunk/DimensionColumnDataChunk.java | 116 --- .../datastore/chunk/DimensionColumnPage.java| 111 +++ .../chunk/impl/AbstractDimensionColumnPage.java | 89 +++ .../chunk/impl/AbstractDimensionDataChunk.java | 95 --- .../impl/ColumnGroupDimensionColumnPage.java| 194 ++ .../impl/ColumnGroupDimensionDataChunk.java | 194 -- .../chunk/impl/DimensionRawColumnChunk.java | 46 +- .../impl/FixedLengthDimensionColumnPage.java| 163 + .../impl/FixedLengthDimensionDataChunk.java | 163 - .../chunk/impl/MeasureRawColumnChunk.java | 26 +- .../impl/VariableLengthDimensionColumnPage.java | 133 .../impl/VariableLengthDimensionDataChunk.java | 140 .../reader/DimensionColumnChunkReader.java | 14 +- .../chunk/reader/MeasureColumnChunkReader.java | 12 +- .../AbstractChunkReaderV2V3Format.java | 34 +- ...mpressedDimensionChunkFileBasedReaderV1.java | 38 +- ...mpressedDimensionChunkFileBasedReaderV2.java | 30 +- ...essedDimChunkFileBasedPageLevelReaderV3.java | 11 +- ...mpressedDimensionChunkFileBasedReaderV3.java | 49 +- .../AbstractMeasureChunkReaderV2V3Format.java | 42 +- ...CompressedMeasureChunkFileBasedReaderV1.java | 16 +- ...CompressedMeasureChunkFileBasedReaderV2.java | 24 +- ...CompressedMeasureChunkFileBasedReaderV3.java | 45 +- ...essedMsrChunkFileBasedPageLevelReaderV3.java | 8 +- .../chunk/store/ColumnPageWrapper.java | 30 +- .../chunk/store/DimensionDataChunkStore.java| 8 +- .../SafeFixedLengthDimensionDataChunkStore.java | 6 +- ...feVariableLengthDimensionDataChunkStore.java | 8 +- ...nsafeFixedLengthDimensionDataChunkStore.java | 10 +- ...afeVariableLengthDimesionDataChunkStore.java | 10 +- .../datastore/columnar/ColumnGroupModel.java| 26 - .../core/datastore/impl/DFSFileHolderImpl.java | 166 - .../core/datastore/impl/DFSFileReaderImpl.java | 155 .../datastore/impl/DefaultFileTypeProvider.java | 16 +- .../core/datastore/impl/FileFactory.java| 4 +- .../core/datastore/impl/FileHolderImpl.java | 224 -- .../core/datastore/impl/FileReaderImpl.java | 215 ++ .../core/datastore/impl/FileTypeInerface.java | 4 +- .../impl/btree/AbstractBTreeLeafNode.java | 60 +- .../impl/btree/BTreeDataRefNodeFinder.java | 6 +- .../datastore/impl/btree/BTreeNonLeafNode.java | 52 +- .../impl/btree/BlockBTreeLeafNode.java | 6 +- .../impl/btree/BlockletBTreeLeafNode.java | 46 +- .../page/encoding/EncodingFactory.java | 8 +- .../server/NonSecureDictionaryServer.java | 1 - .../core/indexstore/BlockletDetailInfo.java | 4 - .../blockletindex/BlockletDataRefNode.java | 228 ++ .../BlockletDataRefNodeWrapper.java | 241 --- .../indexstore/blockletindex/IndexWrapper.java | 2 +- .../blockletindex/SegmentIndexFileStore.java| 7 +- .../core/memory/HeapMemoryAllocator.java| 2 +- .../core/metadata/blocklet/SegmentInfo.java | 19 - .../core/metadata/schema/table/CarbonTable.java | 130 +++- .../schema/table/RelationIdentifier.java| 16 - .../core/metadata/schema/table/TableInfo.java | 6 +- .../schema/table/column/CarbonColumn.java | 2 +- .../schema/table/column/CarbonDimension.java| 12 - .../core/mutate/CarbonUpdateUtil.java | 20 +- .
[35/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading"
Revert "[CARBONDATA-2023][DataLoad] Add size base block allocation in data
loading"
This reverts commit 6dd8b038fc898dbf48ad30adfc870c19eb38e3d0.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3f1d101d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3f1d101d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3f1d101d
Branch: refs/heads/carbonstore-rebase5
Commit: 3f1d101dc79de5680e6104d76625cd4d88d45011
Parents: 880bbce
Author: Jacky Li
Authored: Sat Feb 10 10:34:59 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:04:49 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 -
.../core/datastore/block/TableBlockInfo.java| 29 --
.../carbondata/core/util/CarbonProperties.java | 11 -
docs/useful-tips-on-carbondata.md | 1 -
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 --
.../processing/util/CarbonLoaderUtil.java | 494 +++
.../processing/util/CarbonLoaderUtilTest.java | 125 -
10 files changed, 183 insertions(+), 551 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3f1d101d/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a6bf60f..bcfeba0 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,14 +114,4 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
- /**
- * enable block size based block allocation while loading data. By default,
carbondata assigns
- * blocks to node based on block number. If this option is set to `true`,
carbondata will
- * consider block size first and make sure that all the nodes will process
almost equal size of
- * data. This option is especially useful when you encounter skewed data.
- */
- @CarbonProperty
- public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
- = "carbon.load.skewedDataOptimization.enabled";
- public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3f1d101d/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c0cebe0..a7bfdba 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -100,20 +98,6 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
- /**
- * comparator to sort by block size in descending order.
- * Since each line is not exactly the same, the size of a InputSplit may
differs,
- * so we allow some deviation for these splits.
- */
- public static final Comparator DATA_SIZE_DESC_COMPARATOR =
- new Comparator() {
-@Override public int compare(Distributable o1, Distributable o2) {
- long diff =
- ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
- return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
-}
- };
-
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -450,17 +434,4 @@ public class TableBlockInfo implements Distributable,
Serializable {
public void setDataMapWriterPath(String dataMapWriterPath) {
this.dataMapWriterPath = dataMapWriterPath;
}
-
- @Override
- public String toString() {
-final StringBuilder sb = new StringBuilder("TableBlockInfo{");
-sb.append("filePath='").append(filePa
[02/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove partitionId in CarbonTablePath
[CARBONDATA-1992] Remove partitionId in CarbonTablePath In CarbonTablePath, there is a deprecated partition id which is always 0, it should be removed to avoid confusion. This closes #1765 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c770ff68 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c770ff68 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c770ff68 Branch: refs/heads/carbonstore-rebase5 Commit: c770ff686dd969b7bd7019efaacc908a36c1b9cc Parents: f5df308 Author: Jacky Li Authored: Sat Jan 6 20:28:44 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 18:15:03 2018 +0800 -- .../core/metadata/PartitionMapFileStore.java| 0 .../core/mutate/CarbonUpdateUtil.java | 8 +- .../core/statusmanager/LoadMetadataDetails.java | 1 + .../SegmentUpdateStatusManager.java | 8 +- .../apache/carbondata/core/util/CarbonUtil.java | 6 +- .../core/util/path/CarbonTablePath.java | 55 --- .../CarbonFormatDirectoryStructureTest.java | 4 +- .../hadoop/api/CarbonTableInputFormat.java | 2 +- .../streaming/CarbonStreamRecordWriter.java | 2 +- .../hadoop/test/util/StoreCreator.java | 1 - .../presto/util/CarbonDataStoreCreator.scala| 1 - .../dataload/TestLoadDataGeneral.scala | 2 +- .../InsertIntoCarbonTableTestCase.scala | 4 +- .../dataload/TestBatchSortDataLoad.scala| 3 +- .../dataload/TestDataLoadWithFileName.scala | 2 +- .../dataload/TestGlobalSortDataLoad.scala | 4 +- .../testsuite/datamap/TestDataMapCommand.scala | 34 ++-- .../TestDataLoadingForPartitionTable.scala | 3 +- .../load/DataLoadProcessBuilderOnSpark.scala| 1 - .../load/DataLoadProcessorStepOnSpark.scala | 2 +- .../spark/rdd/AlterTableLoadPartitionRDD.scala | 154 +++ .../carbondata/spark/rdd/CarbonMergerRDD.scala | 11 +- .../spark/rdd/NewCarbonDataLoadRDD.scala| 25 ++- .../org/apache/spark/util/PartitionUtils.scala | 5 +- .../spark/rdd/CarbonDataRDDFactory.scala| 5 +- .../management/CarbonLoadDataCommand.scala | 1 - .../datasources/CarbonFileFormat.scala | 1 - .../partition/TestAlterPartitionTable.scala | 2 +- .../bucketing/TableBucketingTestCase.scala | 2 + .../loading/CarbonDataLoadConfiguration.java| 10 -- .../loading/DataLoadProcessBuilder.java | 1 - .../loading/TableProcessingOperations.java | 3 +- .../loading/model/CarbonLoadModel.java | 73 + .../sort/impl/ParallelReadMergeSorterImpl.java | 4 +- ...arallelReadMergeSorterWithBucketingImpl.java | 15 +- .../UnsafeBatchParallelReadMergeSorterImpl.java | 7 +- ...arallelReadMergeSorterWithBucketingImpl.java | 21 ++- .../CarbonRowDataWriterProcessorStepImpl.java | 33 ++-- .../steps/DataWriterBatchProcessorStepImpl.java | 25 +-- .../steps/DataWriterProcessorStepImpl.java | 22 +-- .../processing/merger/CarbonDataMergerUtil.java | 6 +- .../merger/CompactionResultSortProcessor.java | 8 +- .../merger/RowResultMergerProcessor.java| 2 +- .../partition/spliter/RowResultProcessor.java | 2 +- .../sort/sortdata/SortParameters.java | 16 +- .../store/CarbonFactDataHandlerModel.java | 3 +- .../util/CarbonDataProcessorUtil.java | 20 +-- .../processing/util/CarbonLoaderUtil.java | 12 +- .../processing/util/DeleteLoadFolders.java | 7 +- .../carbondata/processing/StoreCreator.java | 1 - .../carbondata/streaming/StreamHandoffRDD.scala | 1 - .../streaming/StreamSinkFactory.scala | 2 +- .../streaming/CarbonAppendableStreamSink.scala | 8 +- 53 files changed, 285 insertions(+), 366 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c770ff68/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java new file mode 100644 index 000..e69de29 http://git-wip-us.apache.org/repos/asf/carbondata/blob/c770ff68/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index de98fa8..18eae11 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdat
[05/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index f51ced3..6a401d8 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -34,20 +34,16 @@ import
org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import
org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
/**
* Executor class for executing the query on the selected segments to be
merged.
@@ -70,6 +66,9 @@ public class CarbonCompactionExecutor {
*/
private boolean restructuredBlockExists;
+ // converter for UTF8String and decimal conversion
+ private DataTypeConverter dataTypeConverter;
+
/**
* Constructor
*
@@ -82,13 +81,14 @@ public class CarbonCompactionExecutor {
public CarbonCompactionExecutor(Map segmentMapping,
SegmentProperties segmentProperties, CarbonTable carbonTable,
Map> dataFileMetadataSegMapping,
- boolean restructuredBlockExists) {
+ boolean restructuredBlockExists, DataTypeConverter dataTypeConverter) {
this.segmentMapping = segmentMapping;
this.destinationSegProperties = segmentProperties;
this.carbonTable = carbonTable;
this.dataFileMetadataSegMapping = dataFileMetadataSegMapping;
this.restructuredBlockExists = restructuredBlockExists;
-queryExecutorList = new
ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+this.queryExecutorList = new
ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+this.dataTypeConverter = dataTypeConverter;
}
/**
@@ -100,7 +100,9 @@ public class CarbonCompactionExecutor {
List resultList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List list = null;
-queryModel = prepareQueryModel(list);
+queryModel =
carbonTable.createQueryModelWithProjectAllColumns(dataTypeConverter);
+queryModel.setReadPageByPage(enablePageLevelReaderForCompaction());
+queryModel.setForcedDetailRawQuery(true);
// iterate each seg ID
for (Map.Entry taskMap : segmentMapping.entrySet())
{
String segmentId = taskMap.getKey();
@@ -156,7 +158,7 @@ public class CarbonCompactionExecutor {
* @param blockList
* @return
*/
- private CarbonIterator executeBlockList(List
blockList)
+ private CarbonIterator executeBlockList(List
blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
QueryExecutor queryExecutor =
QueryExecutorFactory.getQueryExecutor(queryModel);
@@ -195,48 +197,6 @@ public class CarbonCompactionExecutor {
}
/**
- * Preparing of the query model.
- *
- * @param blockList
- * @return
- */
- private QueryModel prepareQueryModel(List blockList) {
-QueryModel model = new QueryModel();
-model.setTableBlockInfos(blockList);
-model.setForcedDetailRawQuery(true);
-model.setFilterExpressionResolverTree(null);
-model.setConverter(DataTypeUtil.getDataTypeConverter());
-model.setReadPageByPage(enablePageLevelReaderForCompaction());
-
-List dims = new
ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-List dimensions =
-carbonTable.getDimensionByTableName(carbonTable.getTableName());
-for (CarbonDimension dim : dimensions) {
- // check if dimension is deleted
- QueryDimension
[29/50] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8fcde02e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 11b3d43..527452a 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,15 +31,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import
org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import
org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -63,21 +62,15 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
* entry count
*/
private int entryCount;
-
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -100,27 +93,21 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
private int numberOfObjectRead;
- private int nullSetWordsLength;
-
- private Comparator comparator;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator comparator;
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters
parameters) {
// set temp file
this.tempFile = tempFile;
-this.dimCnt = parameters.getDimColCount();
-this.complexCnt = parameters.getComplexDimColCount();
-this.measureCnt = parameters.getMeasureColCount();
-this.isNoDictionaryDimensionColumn =
parameters.getNoDictionaryDimnesionColumn();
-this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-
+this.tableFieldStat = new TableFieldStat(parameters);
+this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.executorService = Executors.newFixedThreadPool(1);
-this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
-comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
+comparator = new
IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -169,11 +156,17 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
+ @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+this.returnRow =
sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+this.numberOfObjectRead++;
+ } catch (IOException e) {
+throw new CarbonSortKeyAndGroupByException("Problems while reading
row", e);
+ }
}
}
@@ -207,63 +200,22 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
}
/**
- * @retu
[36/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bfdf3e35/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 527452a..11b3d43 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,14 +31,15 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import
org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import
org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -62,15 +63,21 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
* entry count
*/
private int entryCount;
+
/**
* return row
*/
- private IntermediateSortTempRow returnRow;
+ private Object[] returnRow;
+ private int dimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private DataType[] measureDataTypes;
private int readBufferSize;
private String compressorName;
- private IntermediateSortTempRow[] currentBuffer;
+ private Object[][] currentBuffer;
- private IntermediateSortTempRow[] backupBuffer;
+ private Object[][] backupBuffer;
private boolean isBackupFilled;
@@ -93,21 +100,27 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
private int numberOfObjectRead;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
- private Comparator comparator;
+ private int nullSetWordsLength;
+
+ private Comparator comparator;
+
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters
parameters) {
// set temp file
this.tempFile = tempFile;
+this.dimCnt = parameters.getDimColCount();
+this.complexCnt = parameters.getComplexDimColCount();
+this.measureCnt = parameters.getMeasureColCount();
+this.isNoDictionaryDimensionColumn =
parameters.getNoDictionaryDimnesionColumn();
+this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-this.tableFieldStat = new TableFieldStat(parameters);
-this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+
this.executorService = Executors.newFixedThreadPool(1);
-comparator = new
IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
+this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
+comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -156,17 +169,11 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
- @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- try {
-this.returnRow =
sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
-this.numberOfObjectRead++;
- } catch (IOException e) {
-throw new CarbonSortKeyAndGroupByException("Problems while reading
row", e);
- }
+ this.returnRow = getRowFromStream();
}
}
@@ -200,22 +207,63 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
}
/**
- * get a
[23/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
[CARBONDATA-2025] Unify all path construction through CarbonTablePath static method Refactory CarbonTablePath: 1.Remove CarbonStorePath and use CarbonTablePath only. 2.Make CarbonTablePath an utility without object creation, it can avoid creating object before using it, thus code is cleaner and GC is less. This closes #1768 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9b9125b6 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9b9125b6 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9b9125b6 Branch: refs/heads/carbonstore-rebase5 Commit: 9b9125b6dfa4373364faca9b605569fb2d775c17 Parents: 55c4e43 Author: Jacky Li Authored: Wed Jan 31 16:14:27 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 19:31:36 2018 +0800 -- .../DictionaryColumnUniqueIdentifier.java | 29 +- .../dictionary/ManageDictionaryAndBTree.java| 13 +- .../core/metadata/AbsoluteTableIdentifier.java | 4 +- .../core/metadata/SegmentFileStore.java | 3 +- .../core/metadata/schema/table/CarbonTable.java | 11 +- .../core/mutate/CarbonUpdateUtil.java | 58 ++-- .../core/scan/executor/util/QueryUtil.java | 7 +- .../scan/executor/util/RestructureUtil.java | 6 +- .../scan/filter/FilterExpressionProcessor.java | 2 +- .../filter/executer/FalseFilterExecutor.java| 8 +- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 1 + ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 1 + ...velRangeLessThanEqualFilterExecuterImpl.java | 1 + .../RowLevelRangeLessThanFiterExecuterImpl.java | 1 + .../FalseConditionalResolverImpl.java | 4 +- .../core/service/CarbonCommonFactory.java | 16 - .../carbondata/core/service/PathService.java| 35 --- .../core/service/impl/PathFactory.java | 50 .../statusmanager/SegmentStatusManager.java | 31 +- .../SegmentUpdateStatusManager.java | 70 ++--- .../apache/carbondata/core/util/CarbonUtil.java | 98 ++ .../util/path/CarbonSharedDictionaryPath.java | 71 - .../core/util/path/CarbonStorePath.java | 71 - .../core/util/path/CarbonTablePath.java | 298 ++- .../dictionary/AbstractDictionaryCacheTest.java | 11 +- .../dictionary/ForwardDictionaryCacheTest.java | 6 +- .../dictionary/ReverseDictionaryCacheTest.java | 6 +- .../reader/CarbonDictionaryReaderImplTest.java | 8 - .../CarbonFormatDirectoryStructureTest.java | 18 +- .../path/CarbonFormatSharedDictionaryTest.java | 44 --- .../writer/CarbonDictionaryWriterImplTest.java | 19 +- .../CarbonBatchSparkStreamingExample.scala | 9 +- .../CarbonStreamSparkStreamingExample.scala | 10 +- .../CarbonStructuredStreamingExample.scala | 11 +- ...CarbonStructuredStreamingWithRowParser.scala | 9 +- .../hadoop/api/CarbonTableInputFormat.java | 8 +- .../streaming/CarbonStreamRecordWriter.java | 6 +- .../carbondata/hadoop/util/SchemaReader.java| 18 +- .../hadoop/test/util/StoreCreator.java | 16 +- .../presto/CarbondataRecordSetProvider.java | 7 +- .../presto/impl/CarbonTableCacheModel.java | 13 +- .../presto/impl/CarbonTableReader.java | 49 ++- .../presto/util/CarbonDataStoreCreator.scala| 10 +- .../sdv/generated/MergeIndexTestCase.scala | 19 +- .../dataload/TestLoadDataGeneral.scala | 11 +- .../InsertIntoCarbonTableTestCase.scala | 8 +- .../createTable/TestCreateTableAsSelect.scala | 2 +- .../datacompaction/DataCompactionLockTest.scala | 6 +- .../MajorCompactionIgnoreInMinorTest.scala | 12 +- .../dataload/TestBatchSortDataLoad.scala| 5 +- .../dataload/TestDataLoadWithFileName.scala | 5 +- .../dataload/TestGlobalSortDataLoad.scala | 8 +- .../dataretention/DataRetentionTestCase.scala | 6 +- .../TestDataLoadingForPartitionTable.scala | 4 +- .../StandardPartitionTableLoadingTestCase.scala | 6 +- .../org/apache/carbondata/api/CarbonStore.scala | 4 +- .../carbondata/spark/CarbonSparkFactory.scala | 2 +- .../spark/DictionaryDetailHelper.scala | 9 +- .../spark/rdd/AlterTableAddColumnRDD.scala | 17 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 6 - .../carbondata/spark/util/CommonUtil.scala | 30 +- .../carbondata/spark/util/DataLoadingUtil.scala | 10 +- .../spark/util/GlobalDictionaryUtil.scala | 16 +- .../command/carbonTableSchemaCommon.scala | 9 +- .../org/apache/spark/util/PartitionUtils.scala | 6 +- .../spark/rdd/AggregateDataMapCompactor.scala | 19 +- .../spark/rdd/CarbonDataRDDFactory.scala| 14 +- .../spark/rdd/CarbonTableCompactor.scala| 8 +- .../CarbonAlterTableCompactionCommand.scala | 13 +- .../management/CarbonLoadDataCommand.scala | 12 +-
[25/50] [abbrv] carbondata git commit: [CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to executor for s3 implementation in cluster mode.
[CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to
executor for s3 implementation in cluster mode.
Problem : hadoopconf was not getting propagated from driver to the executor
that's why load was failing to the distributed environment.
Solution: Setting the Hadoop conf in base class CarbonRDD
How to verify this PR :
Execute the load in the cluster mode It should be a success using location s3.
This closes #1860
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f8841d87
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f8841d87
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f8841d87
Branch: refs/heads/carbonstore-rebase5
Commit: f8841d873d281046657a2afcf6efd343c0f154e4
Parents: 9b9125b
Author: Jatin
Authored: Thu Jan 25 16:53:00 2018 +0530
Committer: Jacky Li
Committed: Sun Mar 4 19:31:39 2018 +0800
--
.../spark/rdd/AlterTableAddColumnRDD.scala | 2 +-
.../spark/rdd/AlterTableDropColumnRDD.scala | 2 +-
.../spark/rdd/CarbonCleanFilesRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadRDD.scala | 2 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 2 +-
.../spark/rdd/CarbonDropTableRDD.scala | 2 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +-
.../spark/rdd/CarbonMergeFilesRDD.scala | 0
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
.../apache/carbondata/spark/rdd/CarbonRDD.scala | 32 ++--
.../spark/rdd/NewCarbonDataLoadRDD.scala| 2 +-
.../carbondata/spark/rdd/SparkDataMapJob.scala | 2 +-
.../apache/spark/rdd/DataLoadCoalescedRDD.scala | 3 +-
.../apache/spark/rdd/UpdateCoalescedRDD.scala | 2 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
16 files changed, 45 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8841d87/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index 56a66b9..7c1edea 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -50,7 +50,7 @@ class AddColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Par
class AlterTableAddColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
identifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
val lockType: String =
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8841d87/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
index 248f351..e14524e 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
@@ -48,7 +48,7 @@ class DropColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Pa
class AlterTableDropColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
carbonTableIdentifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
newColumns.zipWithIndex.map { column =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8841d87/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
index 32523d8..9936a2a 100644
---
a/integration/spark-c
[28/50] [abbrv] carbondata git commit: [HotFix][CheckStyle] Fix import related checkstyle
[HotFix][CheckStyle] Fix import related checkstyle This closes #1952 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0feaeb54 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0feaeb54 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0feaeb54 Branch: refs/heads/carbonstore-rebase5 Commit: 0feaeb545bb9e45a1cca35b75d825d8b86aa938d Parents: 828ae5e Author: xuchuanyin Authored: Thu Feb 8 15:39:45 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 19:44:12 2018 +0800 -- .../core/indexstore/blockletindex/BlockletDataRefNode.java | 2 +- .../org/apache/carbondata/core/memory/HeapMemoryAllocator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/0feaeb54/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java index b8fd6ff..50862a7 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java @@ -33,8 +33,8 @@ import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReade import org.apache.carbondata.core.indexstore.BlockletDetailInfo; import org.apache.carbondata.core.indexstore.FineGrainBlocklet; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; -import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex; +import org.apache.carbondata.core.util.BitSetGroup; /** * wrapper for blocklet data map data http://git-wip-us.apache.org/repos/asf/carbondata/blob/0feaeb54/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java index 53cbb1d..242995b 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java @@ -17,11 +17,11 @@ package org.apache.carbondata.core.memory; -import javax.annotation.concurrent.GuardedBy; import java.lang.ref.WeakReference; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; +import javax.annotation.concurrent.GuardedBy; import org.apache.carbondata.core.util.CarbonProperties;
[42/50] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daecc774/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 11b3d43..527452a 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,15 +31,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import
org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import
org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -63,21 +62,15 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
* entry count
*/
private int entryCount;
-
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -100,27 +93,21 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
private int numberOfObjectRead;
- private int nullSetWordsLength;
-
- private Comparator comparator;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator comparator;
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters
parameters) {
// set temp file
this.tempFile = tempFile;
-this.dimCnt = parameters.getDimColCount();
-this.complexCnt = parameters.getComplexDimColCount();
-this.measureCnt = parameters.getMeasureColCount();
-this.isNoDictionaryDimensionColumn =
parameters.getNoDictionaryDimnesionColumn();
-this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-
+this.tableFieldStat = new TableFieldStat(parameters);
+this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.executorService = Executors.newFixedThreadPool(1);
-this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
-comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
+comparator = new
IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -169,11 +156,17 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
+ @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+this.returnRow =
sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+this.numberOfObjectRead++;
+ } catch (IOException e) {
+throw new CarbonSortKeyAndGroupByException("Problems while reading
row", e);
+ }
}
}
@@ -207,63 +200,22 @@ public class UnsafeSortTempFileChunkHolder implements
SortTempChunkHolder {
}
/**
- * @retu
[18/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after rebasing master
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/03157f91
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/03157f91
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/03157f91
Branch: refs/heads/carbonstore-rebase5
Commit: 03157f91120e3de1e12cca1cf7230aafc6856e0d
Parents: 4628d2e
Author: Jacky Li
Authored: Thu Feb 1 00:25:31 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 18:15:06 2018 +0800
--
.../hadoop/util/CarbonInputFormatUtil.java | 20 +++
.../spark/rdd/NewCarbonDataLoadRDD.scala| 21 ++--
.../org/apache/spark/sql/CarbonSession.scala| 5 ++---
3 files changed, 24 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03157f91/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
--
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 514428b..056c27b 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -22,6 +22,8 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Locale;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -39,6 +41,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
@@ -159,4 +162,21 @@ public class CarbonInputFormatUtil {
String jobtrackerID = createJobTrackerID(date);
return new JobID(jobtrackerID, batch);
}
+
+ public static void setS3Configurations(Configuration hadoopConf) {
+FileFactory.getConfiguration()
+.set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03157f91/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 917fc88..e17824f 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,10 +41,10 @@ import
org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties,
CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.loading.{DataLoadExecutor,
FailureCauses, T
[09/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
new file mode 100644
index 000..fde4e55
--- /dev/null
+++
b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFullScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.TaskMetricsMap;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public class DataBlockIterator extends CarbonIterator> {
+
+ /**
+ * iterator which will be used to iterate over blocklets
+ */
+ private BlockletIterator blockletIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ private ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ private BlockletScanner blockletScanner;
+
+ /**
+ * batch size of result
+ */
+ private int batchSize;
+
+ private ExecutorService executorService;
+
+ private Future future;
+
+ private Future futureIo;
+
+ private BlockletScannedResult scannedResult;
+
+ private BlockExecutionInfo blockExecutionInfo;
+
+ private FileReader fileReader;
+
+ private AtomicBoolean nextBlock;
+
+ private AtomicBoolean nextRead;
+
+ public DataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileReader
fileReader,
+ int batchSize, QueryStatisticsModel queryStatisticsModel,
ExecutorService executorService) {
+this.blockExecutionInfo = blockExecutionInfo;
+this.fileReader = fileReader;
+blockletIterator = new
BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+blockExecutionInfo.getNumberOfBlockToScan());
+if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new BlockletFilterScanner(blockExecutionInfo,
queryStatisticsModel);
+} else {
+ blockletScanner = new BlockletFullScanner(blockExecutionInfo,
queryStatisticsModel);
+}
+this.scannerResultAggregator =
+ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
+this.batchSize = batchSize;
+this.executorService = executorService;
+this.nextBlock = new AtomicBoolean(false);
+this.nextRead = new AtomicBoolean(false);
+ }
+
+ @Override
+ public List next() {
+List collectedResult = null;
+if (updateScanner()) {
+ collectedResult =
this.scannerResultAggregator.collectResultInRow(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+List data = this.scannerResultAggregator
+.collectResultInRow(scannedResult, batchSize -
collectedResul
[07/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 94a041a..b74c279 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -378,7 +378,6 @@ public abstract class AbstractDataFileFooterConverter {
cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
}
info.setColumnCardinality(cardinality);
-info.setNumberOfColumns(segmentInfo.getNum_cols());
return info;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 52305bd..0cc783e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -48,10 +48,10 @@ import
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
@@ -82,7 +82,7 @@ import
org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
@@ -248,16 +248,13 @@ public final class CarbonUtil {
public static ColumnGroupModel getColGroupModel(int[][] columnGroups) {
int[] columnSplit = new int[columnGroups.length];
int noOfColumnStore = columnSplit.length;
-boolean[] columnarStore = new boolean[noOfColumnStore];
for (int i = 0; i < columnGroups.length; i++) {
columnSplit[i] = columnGroups[i].length;
- columnarStore[i] = columnGroups[i].length <= 1;
}
ColumnGroupModel colGroupModel = new ColumnGroupModel();
colGroupModel.setNoOfColumnStore(noOfColumnStore);
colGroupModel.setColumnSplit(columnSplit);
-colGroupModel.setColumnarStore(columnarStore);
colGroupModel.setColumnGroup(columnGroups);
return colGroupModel;
}
@@ -418,7 +415,7 @@ public final class CarbonUtil {
}
}
- public static int getFirstIndexUsingBinarySearch(DimensionColumnDataChunk
dimColumnDataChunk,
+ public static int getFirstIndexUsingBinarySearch(DimensionColumnPage
dimColumnDataChunk,
int low, int high, byte[] compareValue, boolean matchUpLimit) {
int cmpResult = 0;
while (high >= low) {
@@ -457,7 +454,7 @@ public final class CarbonUtil {
* @return the compareValue's range index in the dimColumnDataChunk
*/
public static int[] getRangeIndexUsingBinarySearch(
- DimensionColumnDataChunk dimColumnDataChunk, int low, int high, byte[]
compareValue) {
+ DimensionColumnPage dimColumnDataChunk, int low, int high, byte[]
compareValue) {
int[] rangeIndex = new int[2];
int cmpResult = 0;
@@ -551,7 +548,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextLesserValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue) {
while (currentIndex - 1 >= 0
&& dimColumnDataChunk.compareTo(currentIndex - 1, compareValue) >= 0) {
--currentIndex;
@@ -571,7 +568,7 @@ public
[41/50] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
[CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module To make assembling JAR of store-sdk module, it should not depend on carbon-spark module This closes #1970 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83df87dd Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83df87dd Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83df87dd Branch: refs/heads/carbonstore-rebase5 Commit: 83df87ddd74eda674a7343fd3f39e6c81650c0cf Parents: a950863 Author: Jacky Li Authored: Sun Feb 11 21:37:04 2018 +0800 Committer: Jacky Li Committed: Sun Mar 4 20:08:28 2018 +0800 -- .../java/org/apache/carbondata/common/Maps.java | 39 ++ .../org/apache/carbondata/common/Strings.java | 3 + .../exceptions/TableStatusLockException.java| 34 ++ .../sql/InvalidLoadOptionException.java | 33 + .../sql/MalformedCarbonCommandException.java| 75 +++ .../sql/MalformedDataMapCommandException.java | 37 ++ .../exceptions/sql/NoSuchDataMapException.java | 39 ++ .../carbondata/core/datamap/TableDataMap.java | 5 +- .../exception/ConcurrentOperationException.java | 50 ++ .../statusmanager/SegmentStatusManager.java | 124 .../carbondata/core/util/DeleteLoadFolders.java | 210 +++ .../preaggregate/TestPreAggCreateCommand.scala | 2 +- .../preaggregate/TestPreAggregateDrop.scala | 2 +- .../timeseries/TestTimeSeriesCreateTable.scala | 2 +- .../timeseries/TestTimeSeriesDropSuite.scala| 2 +- .../TestTimeseriesTableSelection.scala | 2 +- .../TestDataLoadWithColumnsMoreThanSchema.scala | 3 +- .../dataload/TestGlobalSortDataLoad.scala | 2 +- .../TestLoadDataWithDiffTimestampFormat.scala | 2 +- .../TestLoadDataWithFileHeaderException.scala | 11 +- ...ataWithMalformedCarbonCommandException.scala | 3 +- .../testsuite/dataload/TestLoadOptions.scala| 2 +- .../dataload/TestTableLevelBlockSize.scala | 4 +- .../testsuite/datamap/TestDataMapCommand.scala | 2 +- .../dataretention/DataRetentionTestCase.scala | 2 +- .../spark/testsuite/datetype/DateTypeTest.scala | 2 +- .../testsuite/sortcolumns/TestSortColumns.scala | 3 +- integration/spark-common/pom.xml| 5 - .../exception/ConcurrentOperationException.java | 38 -- .../MalformedCarbonCommandException.java| 69 --- .../MalformedDataMapCommandException.java | 32 - .../spark/exception/NoSuchDataMapException.java | 33 - .../org/apache/carbondata/api/CarbonStore.scala | 3 +- .../spark/CarbonColumnValidator.scala | 8 +- .../carbondata/spark/load/ValidateUtil.scala| 72 --- .../carbondata/spark/rdd/CarbonMergerRDD.scala | 6 +- .../carbondata/spark/util/CommonUtil.scala | 70 +-- .../carbondata/spark/util/DataLoadingUtil.scala | 610 --- .../spark/util/GlobalDictionaryUtil.scala | 2 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +- .../spark/rdd/CarbonDataRDDFactory.scala| 4 +- .../spark/rdd/CarbonTableCompactor.scala| 2 +- .../org/apache/spark/sql/CarbonSource.scala | 2 +- .../datamap/CarbonCreateDataMapCommand.scala| 2 +- .../datamap/CarbonDropDataMapCommand.scala | 2 +- .../CarbonAlterTableCompactionCommand.scala | 13 +- .../management/CarbonLoadDataCommand.scala | 17 +- .../CarbonProjectForDeleteCommand.scala | 2 +- .../CarbonProjectForUpdateCommand.scala | 2 +- .../command/mutation/IUDCommonUtil.scala| 2 +- .../CreatePreAggregateTableCommand.scala| 7 +- .../preaaggregate/PreAggregateUtil.scala| 2 +- .../schema/CarbonAlterTableRenameCommand.scala | 3 +- .../command/timeseries/TimeSeriesUtil.scala | 2 +- .../datasources/CarbonFileFormat.scala | 14 +- .../sql/execution/strategy/DDLStrategy.scala| 2 +- .../strategy/StreamingTableStrategy.scala | 2 +- .../execution/command/CarbonHiveCommands.scala | 2 +- .../sql/parser/CarbonSpark2SqlParser.scala | 2 +- .../spark/sql/parser/CarbonSparkSqlParser.scala | 2 +- .../org/apache/spark/util/AlterTableUtil.scala | 2 +- .../org/apache/spark/util/TableAPIUtil.scala| 2 +- .../spark/sql/hive/CarbonSessionState.scala | 7 +- .../segmentreading/TestSegmentReading.scala | 2 +- .../spark/util/AllDictionaryTestCase.scala | 4 +- .../util/ExternalColumnDictionaryTestCase.scala | 6 +- .../TestStreamingTableOperation.scala | 4 +- .../bucketing/TableBucketingTestCase.scala | 2 +- .../vectorreader/AddColumnTestCases.scala | 2 +- .../loading/model/CarbonLoadModel.java | 14 +- .../loading/model/CarbonLoadModelBuilder.java | 322 ++ .../processing/loading/model/LoadOption.java| 251 .../processing/util/CarbonLo
[01/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove partitionId in CarbonTablePath [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/carbonstore-rebase5 c305f309e -> 7f92fde49 (forced update)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c770ff68/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
index f030d52..369c1f2 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -59,13 +60,11 @@ public class DataWriterBatchProcessorStepImpl extends
AbstractDataLoadProcessorS
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier,
String partitionId) {
-String[] storeLocation = CarbonDataProcessorUtil
-.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
-tableIdentifier.getTableName(),
String.valueOf(configuration.getTaskNo()), partitionId,
-configuration.getSegmentId() + "", false, false);
-CarbonDataProcessorUtil.createLocations(storeLocation);
-return storeLocation;
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
+return CarbonDataProcessorUtil.getLocalDataFolderLocation(
+tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(),
+String.valueOf(configuration.getTaskNo()),
+configuration.getSegmentId(), false, false);
}
@Override public Iterator[] execute() throws
CarbonDataLoadingException {
@@ -75,18 +74,19 @@ public class DataWriterBatchProcessorStepImpl extends
AbstractDataLoadProcessorS
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-
.recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+
.recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+ CarbonDataProcessorUtil.createLocations(storeLocation);
for (Iterator iterator : iterators) {
-String[] storeLocation = getStoreLocation(tableIdentifier,
String.valueOf(i));
int k = 0;
while (iterator.hasNext()) {
CarbonRowBatch next = iterator.next();
// If no rows from merge sorter, then don't create a file in fact
column handler
if (next.hasNext()) {
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
-.createCarbonFactDataHandlerModel(configuration,
storeLocation, i, k++);
+.createCarbonFactDataHandlerModel(configuration,
storeLocation, 0, k++);
CarbonFactHandler dataHandler = CarbonFactHandlerFactory
.createCarbonFactHandler(model,
CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
dataHandler.initialise();
@@ -119,10 +119,11 @@ public class DataWriterBatchProcessorStepImpl extends
AbstractDataLoadProcessorS
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-.recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+
.recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
-.recordMdkGenerateTotalTime(configuration.getPartitionId(),
System.currentTimeMillis());
+.recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c770ff68/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
--
diff --git
a/pr
[46/50] [abbrv] carbondata git commit: [CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data loading
[CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data
loading
Enhance data loading performance by specifying sort column bounds
1. Add row range number during convert-process-step
2. Dispatch rows to each sorter by range number
3. Sort/Write process step can be done concurrently in each range
4. Since all sorttemp files will be written in one folders, we add range
number to the file name to distingush them
Tests added and docs updated
This closes #1953
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/49d06c20
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/49d06c20
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/49d06c20
Branch: refs/heads/carbonstore-rebase5
Commit: 49d06c205a06817139fe6b0bb99240013d33a92d
Parents: 8996cd4
Author: xuchuanyin
Authored: Tue Feb 13 10:58:06 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:08:30 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/row/CarbonRow.java | 10 +-
.../ThriftWrapperSchemaConverterImpl.java | 2 +-
.../core/metadata/schema/BucketingInfo.java | 24 +-
.../core/metadata/schema/ColumnRangeInfo.java | 29 ++
.../metadata/schema/SortColumnRangeInfo.java| 83 +
docs/data-management-on-carbondata.md | 11 +
.../TestLoadDataWithSortColumnBounds.scala | 348 +++
.../carbondata/spark/rdd/CarbonScanRDD.scala| 2 +-
.../carbondata/spark/rdd/PartitionDropper.scala | 2 +-
.../spark/rdd/PartitionSplitter.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../loading/CarbonDataLoadConfiguration.java| 11 +
.../loading/DataLoadProcessBuilder.java | 77 +++-
.../loading/converter/RowConverter.java | 2 +-
.../converter/impl/RowConverterImpl.java| 5 +
.../loading/model/CarbonLoadModel.java | 14 +
.../loading/model/CarbonLoadModelBuilder.java | 1 +
.../processing/loading/model/LoadOption.java| 1 +
.../partition/impl/HashPartitionerImpl.java | 10 +-
.../partition/impl/RangePartitionerImpl.java| 71
.../partition/impl/RawRowComparator.java| 63
.../processing/loading/sort/SorterFactory.java | 16 +-
...arallelReadMergeSorterWithBucketingImpl.java | 272 ---
...allelReadMergeSorterWithColumnRangeImpl.java | 289 +++
...arallelReadMergeSorterWithBucketingImpl.java | 263 --
...allelReadMergeSorterWithColumnRangeImpl.java | 293
.../loading/sort/unsafe/UnsafeSortDataRows.java | 6 +-
.../unsafe/merger/UnsafeIntermediateMerger.java | 6 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 11 +-
.../steps/DataConverterProcessorStepImpl.java | 102 +-
...ConverterProcessorWithBucketingStepImpl.java | 161 -
.../steps/DataWriterProcessorStepImpl.java | 70 +++-
.../SingleThreadFinalSortFilesMerger.java | 3 +-
.../processing/sort/sortdata/SortDataRows.java | 11 +-
.../sortdata/SortIntermediateFileMerger.java| 6 +-
.../sort/sortdata/SortParameters.java | 10 +
.../store/CarbonFactDataHandlerColumnar.java| 6 +-
39 files changed, 1558 insertions(+), 750 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/49d06c20/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a6bf60f..8ff8dc4 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -124,4 +124,14 @@ public final class CarbonLoadOptionConstants {
public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
= "carbon.load.skewedDataOptimization.enabled";
public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
+
+ /**
+ * field delimiter for each field in one bound
+ */
+ public static final String SORT_COLUMN_BOUNDS_FIELD_DELIMITER = ",";
+
+ /**
+ * row delimiter for each sort column bounds
+ */
+ public static final String SORT_COLUMN_BOUNDS_ROW_DELIMITER = ";";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/49d06c20/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
--
diff --git
a/core/src/main/java/org/apache/carbondata
[27/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap. FG Query flow as follows. 1.The user can add FG datamap to any table and implement there interfaces. 2. Any filter query which hits the table with datamap will call prune method of FGdatamap. 3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well. 4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits. 5. Based on the splits scanrdd schedule the tasks. 6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it. This closes #1471 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/828ae5ec Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/828ae5ec Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/828ae5ec Branch: refs/heads/carbonstore-rebase5 Commit: 828ae5ecef68e97824693c60743a781c22f08d34 Parents: 3a0a9e6 Author: ravipesala Authored: Wed Nov 15 19:48:40 2017 +0530 Committer: Jacky Li Committed: Sun Mar 4 19:44:10 2018 +0800 -- .../carbondata/core/datamap/DataMapMeta.java| 8 +- .../core/datamap/DataMapStoreManager.java | 30 +- .../carbondata/core/datamap/DataMapType.java| 21 + .../carbondata/core/datamap/TableDataMap.java | 33 +- .../core/datamap/dev/AbstractDataMapWriter.java | 110 + .../core/datamap/dev/BlockletSerializer.java| 57 +++ .../carbondata/core/datamap/dev/DataMap.java| 5 +- .../core/datamap/dev/DataMapFactory.java| 14 +- .../core/datamap/dev/DataMapWriter.java | 57 --- .../cgdatamap/AbstractCoarseGrainDataMap.java | 24 + .../AbstractCoarseGrainDataMapFactory.java | 34 ++ .../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 + .../AbstractFineGrainDataMapFactory.java| 38 ++ .../carbondata/core/datastore/DataRefNode.java | 6 + .../core/datastore/block/TableBlockInfo.java| 10 + .../impl/btree/AbstractBTreeLeafNode.java | 5 + .../datastore/impl/btree/BTreeNonLeafNode.java | 5 + .../carbondata/core/indexstore/Blocklet.java| 30 +- .../indexstore/BlockletDataMapIndexStore.java | 6 - .../core/indexstore/BlockletDetailsFetcher.java | 9 + .../core/indexstore/ExtendedBlocklet.java | 17 + .../core/indexstore/FineGrainBlocklet.java | 120 + .../indexstore/SegmentPropertiesFetcher.java| 3 +- .../blockletindex/BlockletDataMap.java | 18 +- .../blockletindex/BlockletDataMapFactory.java | 66 ++- .../blockletindex/BlockletDataRefNode.java | 27 +- .../indexstore/blockletindex/IndexWrapper.java | 18 + .../core/indexstore/schema/FilterType.java | 24 - .../executer/ExcludeFilterExecuterImpl.java | 3 + .../executer/IncludeFilterExecuterImpl.java | 3 + .../scanner/impl/BlockletFilterScanner.java | 2 + .../apache/carbondata/core/util/CarbonUtil.java | 97 datamap/examples/pom.xml| 2 +- .../datamap/examples/MinMaxDataMap.java | 33 +- .../datamap/examples/MinMaxDataMapFactory.java | 67 +-- .../datamap/examples/MinMaxDataWriter.java | 36 +- .../examples/MinMaxIndexBlockDetails.java | 13 - .../carbondata/hadoop/CarbonInputSplit.java | 21 +- .../hadoop/api/CarbonTableInputFormat.java | 17 +- .../testsuite/datamap/CGDataMapTestCase.scala | 361 +++ .../testsuite/datamap/DataMapWriterSuite.scala | 43 +- .../testsuite/datamap/FGDataMapTestCase.scala | 440 +++ .../TestInsertAndOtherCommandConcurrent.scala | 21 +- .../carbondata/spark/rdd/CarbonScanRDD.scala| 7 +- .../TestStreamingTableOperation.scala | 5 +- .../datamap/DataMapWriterListener.java | 57 ++- .../store/CarbonFactDataHandlerModel.java | 10 +- .../store/writer/AbstractFactDataWriter.java| 128 +- 48 files changed, 1784 insertions(+), 401 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/828ae5ec/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java index 7746acf..dd15ccb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java @@ -19,15 +19,15 @@ package org.apache.carbondata
[11/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index de97e82..540607d 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -58,7 +58,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,20 +68,20 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List dimColEvaluatorInfoList;
- protected List msrColEvalutorInfoList;
+ List dimColEvaluatorInfoList;
+ List msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
protected SegmentProperties segmentProperties;
/**
* it has index at which given dimension is stored in file
*/
- protected int[] dimensionBlocksIndex;
+ int[] dimensionChunkIndex;
/**
* it has index at which given measure is stored in file
*/
- protected int[] measureBlocksIndex;
+ int[] measureChunkIndex;
private Map complexDimensionInfoMap;
@@ -89,18 +89,18 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
* flag to check whether the filter dimension is present in current block
list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean[] isDimensionPresentInCurrentBlock;
+ boolean[] isDimensionPresentInCurrentBlock;
/**
* flag to check whether the filter measure is present in current block list
of measures.
* Applicable for restructure scenarios
*/
- protected boolean[] isMeasurePresentInCurrentBlock;
+ boolean[] isMeasurePresentInCurrentBlock;
/**
* is dimension column data is natural sorted
*/
- protected boolean isNaturalSorted;
+ boolean isNaturalSorted;
/**
* date direct dictionary generator
@@ -124,10 +124,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new
boolean[dimColEvaluatorInfoList.size()];
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
- this.dimensionBlocksIndex = new int[]{0};
+ this.dimensionChunkIndex = new int[]{0};
}
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new
ArrayList(20);
@@ -136,10 +136,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new
boolean[msrColEvalutorInfoList.size()];
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
- this.measureBlocksIndex = new int[] {0};
+ this.measureChunkInde
[08/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
index 553f85e..773fbd7 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
@@ -17,20 +17,15 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
public class PartitionSpliterRawResultIterator extends
CarbonIterator {
- private CarbonIterator iterator;
- private BatchResult batch;
+ private CarbonIterator iterator;
+ private RowBatch batch;
private int counter;
- private static final LogService LOGGER =
-
LogServiceFactory.getLogService(PartitionSpliterRawResultIterator.class.getName());
-
- public PartitionSpliterRawResultIterator(CarbonIterator
iterator) {
+ public PartitionSpliterRawResultIterator(CarbonIterator iterator) {
this.iterator = iterator;
}
@@ -65,7 +60,7 @@ public class PartitionSpliterRawResultIterator extends
CarbonIterator
* @param batch
* @return
*/
- private boolean checkBatchEnd(BatchResult batch) {
+ private boolean checkBatchEnd(RowBatch batch) {
return !(counter < batch.getSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
index 70d0958..1dd1595 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -37,7 +37,7 @@ public class RawResultIterator extends
CarbonIterator {
/**
* Iterator of the Batch raw result.
*/
- private CarbonIterator detailRawQueryResultIterator;
+ private CarbonIterator detailRawQueryResultIterator;
/**
* Counter to maintain the row counter.
@@ -55,9 +55,9 @@ public class RawResultIterator extends
CarbonIterator {
/**
* batch of the result.
*/
- private BatchResult batch;
+ private RowBatch batch;
- public RawResultIterator(CarbonIterator
detailRawQueryResultIterator,
+ public RawResultIterator(CarbonIterator
detailRawQueryResultIterator,
SegmentProperties sourceSegProperties, SegmentProperties
destinationSegProperties) {
this.detailRawQueryResultIterator = detailRawQueryResultIterator;
this.sourceSegProperties = sourceSegProperties;
@@ -155,7 +155,7 @@ public class RawResultIterator extends
CarbonIterator {
* @param batch
* @return
*/
- private boolean checkIfBatchIsProcessedCompletely(BatchResult batch) {
+ private boolean checkIfBatchIsProcessedCompletely(RowBatch batch) {
if (counter < batch.getSize()) {
return false;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
index cc9710e..c7cb00d 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -35,10 +35,12 @@ public class VectorDetailQueryResultIterator ext
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b9125b6/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 9799ac2..b7b5e43 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -22,16 +22,14 @@ import
org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.hadoop.fs.Path;
-
/**
* Helps to get Table content paths.
*/
-public class CarbonTablePath extends Path {
+public class CarbonTablePath {
private static final String METADATA_DIR = "Metadata";
private static final String DICTIONARY_EXT = ".dict";
@@ -54,19 +52,10 @@ public class CarbonTablePath extends Path {
private static final String STREAMING_LOG_DIR = "log";
private static final String STREAMING_CHECKPOINT_DIR = "checkpoint";
- private String tablePath;
- private CarbonTableIdentifier carbonTableIdentifier;
-
/**
- * structure CarbonTablePath object to manage table paths
- *
- * @param carbonTableIdentifier identifier of carbon table that the segment
belong to
- * @param tablePathString the store path of the segment
+ * This class provides static utility only.
*/
- public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String
tablePathString) {
-super(tablePathString);
-this.carbonTableIdentifier = carbonTableIdentifier;
-this.tablePath = tablePathString;
+ private CarbonTablePath() {
}
/**
@@ -130,40 +119,21 @@ public class CarbonTablePath extends Path {
}
/**
- * gets table path
+ * Return absolute path of dictionary file
*/
- public String getPath() {
-return tablePath;
+ public static String getDictionaryFilePath(String tablePath, String
columnId) {
+return getMetadataPath(tablePath) + File.separator +
getDictionaryFileName(columnId);
}
/**
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of dictionary file
*/
- public String getDictionaryFilePath(String columnId) {
-return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
- }
-
- /**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
- */
- public String getDictionaryFilePath(String dictionaryPath, String columnId) {
+ public static String getExternalDictionaryFilePath(String dictionaryPath,
String columnId) {
return dictionaryPath + File.separator + getDictionaryFileName(columnId);
}
/**
- * This method will return the metadata directory location for a table
- *
- * @return
- */
- public String getMetadataDirectoryPath() {
-return getMetaDataDir();
- }
-
- /**
- * Return metadata path based on `tablePath`
+ * Return metadata path
*/
public static String getMetadataPath(String tablePath) {
return tablePath + File.separator + METADATA_DIR;
@@ -184,67 +154,42 @@ public class CarbonTablePath extends Path {
}
/**
- * @param columnId unique column identifier
- * @return absolute path of dictionary meta file
+ * Return absolute path of dictionary meta file
*/
- public String getDictionaryMetaFilePath(String columnId) {
-return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
- }
-
- /**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
- */
- public String getDictionaryMetaFilePath(String dictionaryPath, String
columnId) {
+ public static String getExternalDictionaryMetaFilePath(String
dictionaryPath, String columnId) {
return dictionaryPath + File.separator + columnId + DICTIONARY_META_EXT;
}
/**
- * @param columnId unique column identifier
- * @return absolute path of sort index file
+ * Return absolute path of dictionary meta file
*/
- public String getSortIndexFilePath(String columnId) {
-return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+ public static String getDictionaryMetaFilePath(String tablePath, String
columnId) {
+return getMetadataPath(tablePath) + File.separator + columnId +
DICTIONARY_META_EXT;
}
/**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary
[14/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
index 0cb2918..099fffd 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
@@ -29,31 +29,12 @@ public class SegmentInfo implements Serializable {
private static final long serialVersionUID = -174987462709431L;
/**
- * number of column in the segment
- */
- private int numberOfColumns;
-
- /**
* cardinality of each columns
* column which is not participating in the multidimensional key cardinality
will be -1;
*/
private int[] columnCardinality;
/**
- * @return the numberOfColumns
- */
- public int getNumberOfColumns() {
-return numberOfColumns;
- }
-
- /**
- * @param numberOfColumns the numberOfColumns to set
- */
- public void setNumberOfColumns(int numberOfColumns) {
-this.numberOfColumns = numberOfColumns;
- }
-
- /**
* @return the columnCardinality
*/
public int[] getColumnCardinality() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6036569..d17d865 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -19,7 +19,13 @@ package org.apache.carbondata.core.metadata.schema.table;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -33,7 +39,10 @@ import
org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import
org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -136,10 +145,7 @@ public class CarbonTable implements Serializable {
/**
* During creation of TableInfo from hivemetastore the DataMapSchemas and
the columns
* DataTypes are not converted to the appropriate child classes.
- *
* This method will cast the same to the appropriate classes
- *
- * @param tableInfo
*/
public static void updateTableInfo(TableInfo tableInfo) {
List dataMapSchemas = new ArrayList<>();
@@ -153,8 +159,9 @@ public class CarbonTable implements Serializable {
}
tableInfo.setDataMapSchemaList(dataMapSchemas);
for (ColumnSchema columnSchema :
tableInfo.getFactTable().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(),
columnSchema.getScale()));
}
List childSchema = tableInfo.getDataMapSchemaList();
for (DataMapSchema dataMapSchema : childSchema) {
@@ -168,10 +175,11 @@ public class CarbonTable implements Serializable {
}
}
if (tableInfo.getFactTable().getBucketingInfo() != null) {
- for (ColumnSchema columnSchema : tableInfo.getFactTable()
- .getBucketingInfo().getListOfColumns()) {
-
columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
-columnSchema.getPrecision(), columnSchema.getScale()));
+ for (ColumnSchema columnSchema :
+ tableInfo.getFactTable().getBucketingInfo().getListOfColumns()) {
+columnSchema.setDataType(
+DataTypeUtil.valueOf(
+columnSchema.getDataType(), columnSchema.getPrecision(),
[37/50] [abbrv] carbondata git commit: Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row"
Revert "[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort
temp row"
This reverts commit de92ea9a123b17d903f2d1d4662299315c792954.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bfdf3e35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bfdf3e35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bfdf3e35
Branch: refs/heads/carbonstore-rebase5
Commit: bfdf3e354b83b173a3325204074bdee3f72cd71a
Parents: 3f1d101
Author: Jacky Li
Authored: Sat Feb 10 20:11:25 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:04:49 2018 +0800
--
.../carbondata/core/util/NonDictionaryUtil.java | 67 ++-
.../presto/util/CarbonDataStoreCreator.scala| 1 +
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../loading/row/IntermediateSortTempRow.java| 117 -
.../loading/sort/SortStepRowHandler.java| 466 ---
.../loading/sort/SortStepRowUtil.java | 103
.../sort/unsafe/UnsafeCarbonRowPage.java| 331 +++--
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 ++-
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 +++
.../UnsafeRowComparatorForNormalDims.java | 59 ---
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 --
.../merger/UnsafeIntermediateFileMerger.java| 118 -
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 +
.../sort/sortdata/IntermediateFileMerger.java | 95 +++-
.../IntermediateSortTempRowComparator.java | 73 ---
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94
.../sortdata/RowComparatorForNormalDims.java| 62 +++
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +++-
.../sort/sortdata/SortTempFileChunkHolder.java | 174 +--
.../sort/sortdata/TableFieldStat.java | 176 ---
28 files changed, 1294 insertions(+), 1186 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bfdf3e35/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index fca1244..d6ecfbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,26 +82,18 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required dictionary Dimension from obj []
+ * Method to get the required Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static int getDictDimension(int index, Object[] row) {
-int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+ public static Integer getDimension(int index, Object[] row) {
+
+Integer[] dimensions = (Integer[])
row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+
return dimensions[index];
- }
- /**
- * Method to get the required non-dictionary & complex from 3-parted row
- * @param index
- * @param row
- * @return
- */
- public static byte[] getNoDictOrComplex(int index, Object[] row) {
-byte[][] nonDictArray = (byte[][])
row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-return nonDictArray[index];
}
/**
@@ -116,11 +108,60 @@ public class NonDictionaryUtil {
return measures[index];
}
+ public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
+
+return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+ }
+
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][]
byteBufferArr,
Object[] measureArray) {
+
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
+
+ /**
+ * This method will extract the single dimension from the complete high card
dims byte[].+ *
+ * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
+ *
+ * @param highCardArr
+ * @param index
+ * @param highCardinalityCount
+ * @param outBuffer
+ */
+ public static void extractSingleHighCardDims(byte[] highCardArr, int index,
+ int highCardin
[15/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 8c8d08f..a689d8e 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -124,22 +124,22 @@ public class UnsafeFixedLengthDimensionDataChunkStore
/**
* to compare the two byte array
*
- * @param indexindex of first byte array
+ * @param rowIdindex of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// based on index we need to calculate the actual position in memory block
-index = index * columnValueSize;
+rowId = rowId * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
compareResult = (CarbonUnsafe.getUnsafe()
- .getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + index)
+ .getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + rowId)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
break;
}
- index++;
+ rowId++;
}
return compareResult;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 36b2bd8..e1eb378 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -189,11 +189,11 @@ public class UnsafeVariableLengthDimesionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -201,13 +201,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// else subtract the current row offset
// with complete data length get the offset of set of data
int currentDataOffset =
CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
-dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((long)index
+dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((long) rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
-if (index < numberOfRows - 1) {
+if (rowId < numberOfRows - 1) {
int OffsetOfNextdata =
CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((index + 1)
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
index 74d268a..e2a4161 100644
---
a/core/src/main/java/org/apache/
[44/50] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e83d910b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e83d910b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e83d910b
Branch: refs/heads/carbonstore-rebase5
Commit: e83d910b4c9085c0dc6c062a3c05dcd9059e1978
Parents: daecc77
Author: xuchuanyin
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:08:30 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/block/TableBlockInfo.java| 29 ++
.../carbondata/core/util/CarbonProperties.java | 11 +
docs/useful-tips-on-carbondata.md | 1 +
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 ++
.../processing/util/CarbonLoaderUtil.java | 480 ---
.../processing/util/CarbonLoaderUtilTest.java | 125 +
10 files changed, 545 insertions(+), 175 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e83d910b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index bcfeba0..a6bf60f 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,4 +114,14 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
+ /**
+ * enable block size based block allocation while loading data. By default,
carbondata assigns
+ * blocks to node based on block number. If this option is set to `true`,
carbondata will
+ * consider block size first and make sure that all the nodes will process
almost equal size of
+ * data. This option is especially useful when you encounter skewed data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
+ = "carbon.load.skewedDataOptimization.enabled";
+ public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e83d910b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a7bfdba..c0cebe0 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -98,6 +100,20 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
+ /**
+ * comparator to sort by block size in descending order.
+ * Since each line is not exactly the same, the size of a InputSplit may
differs,
+ * so we allow some deviation for these splits.
+ */
+ public static final Comparator DATA_SIZE_DESC_COMPARATOR =
+ new Comparator() {
+@Override public int compare(Distributable o1, Distributable o2) {
+ long diff =
+ ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
+ return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
+}
+ };
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -434,4 +450,17 @@ public class TableBlockInfo implements Distributable,
Serializable {
[13/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 69f5ceb..22d1df1 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -43,10 +43,9 @@ import
org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -64,8 +63,8 @@ import
org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -121,7 +120,6 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
queryProperties.queryStatisticsRecorder =
CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
-QueryUtil.resolveQueryModel(queryModel);
QueryStatistic queryStatistic = new QueryStatistic();
// sort the block info
// so block will be loaded in sorted order this will be required for
@@ -168,12 +166,12 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR,
System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
// calculating the total number of aggeragted columns
-int measureCount = queryModel.getQueryMeasures().size();
+int measureCount = queryModel.getProjectionMeasures().size();
int currentIndex = 0;
DataType[] dataTypes = new DataType[measureCount];
-for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+for (ProjectionMeasure carbonMeasure : queryModel.getProjectionMeasures())
{
// adding the data type and aggregation type of all the measure this
// can be used
// to select the aggregator
@@ -198,9 +196,11 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
queryStatistic = new QueryStatistic();
// dictionary column unique column id to dictionary mapping
// which will be used to get column actual data
-queryProperties.columnToDictionayMapping = QueryUtil
-.getDimensionDictionaryDetail(queryModel.getQueryDimension(),
-queryProperties.complexFilterDimension,
queryModel.getAbsoluteTableIdentifier(),
+queryProperties.columnToDictionayMapping =
+QueryUtil.getDimensionDictionaryDetail(
+queryModel.getProjectionDimensions(),
+queryProperties.complexFilterDimension,
+queryModel.getAbsoluteTableIdentifier(),
tableProvider);
queryStatistic
.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY,
System.currentTimeMillis());
@@ -263,8 +263,8 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
// and query will be executed based on that infos
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
- BlockletDataRefNodeWrapper dataRefNode =
- (BlockletDataRefNodeWrapper) abstractIndex.getDataRefNode();
+ BlockletDataRefNode dataRefNode =
+ (BlockletDataRefNode) abstractIndex.getDataRefNode();
blockExec
[31/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after rebasing master
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ef812484
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ef812484
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ef812484
Branch: refs/heads/carbonstore-rebase5
Commit: ef812484f159945cd7e8921965698f67a6d53a52
Parents: 8fcde02
Author: Jacky Li
Authored: Fri Feb 9 01:39:20 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 19:52:34 2018 +0800
--
.../filter/executer/FalseFilterExecutor.java| 10 +
.../core/util/path/CarbonTablePath.java | 23 +++-
.../CarbonAlterTableDataTypeChangeCommand.scala | 7 +++---
.../schema/CarbonAlterTableRenameCommand.scala | 3 ++-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 9 +---
.../processing/util/CarbonLoaderUtil.java | 4 ++--
6 files changed, 28 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef812484/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
index 1c918bc..ee3b0fc 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
@@ -44,17 +44,19 @@ public class FalseFilterExecutor implements FilterExecuter {
return group;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][]
blockMinValue) {
-
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue)
{
return new BitSet();
}
- @Override public void readColumnChunks(RawBlockletColumnChunks
blockChunkHolder) {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks blockChunkHolder) {
// Do Nothing
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef812484/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index b7b5e43..50c5a31 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -36,13 +36,13 @@ public class CarbonTablePath {
private static final String DICTIONARY_META_EXT = ".dictmeta";
private static final String SORT_INDEX_EXT = ".sortindex";
private static final String SCHEMA_FILE = "schema";
- public static final String TABLE_STATUS_FILE = "tablestatus";
private static final String FACT_DIR = "Fact";
private static final String SEGMENT_PREFIX = "Segment_";
private static final String PARTITION_PREFIX = "Part";
private static final String DATA_PART_PREFIX = "part-";
private static final String BATCH_PREFIX = "_batchno";
+ public static final String TABLE_STATUS_FILE = "tablestatus";
public static final String CARBON_DATA_EXT = ".carbondata";
public static final String INDEX_FILE_EXT = ".carbonindex";
public static final String MERGE_INDEX_FILE_EXT = ".carbonindexmerge";
@@ -140,20 +140,6 @@ public class CarbonTablePath {
}
/**
- * Return metadata path based on `tablePath`
- */
- public static String getTableStatusPath(String tablePath) {
-return getMetadataPath(tablePath) + File.separator + TABLE_STATUS_FILE;
- }
-
- /**
- * Return table status file path based on `tablePath`
- */
- public static String getTableStatusFilePath(String tablePath) {
-return getMetadataPath(tablePath) + CarbonCommonConstants.FILE_SEPARATOR +
TABLE_STATUS_FILE;
- }
-
- /**
* Return absolute path of dictionary meta file
*/
public static String getExternalDictionaryMetaFilePath(String
dictionaryPath, String columnId) {
@@ -213,6 +199,13 @@ public class CarbonTablePath {
}
}
+ /**
+ * Return absolute path of table status file
+ */
+ public static String getTableStatusFilePath(String tablePath) {
+return getMetadataPath(tablePath) + File.separator + TABLE_STATUS_FILE;
+ }
+
public static String getTableStatusFilePathWithUUID(
[40/50] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
http://git-wip-us.apache.org/repos/asf/carbondata/blob/83df87dd/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 8d394db..e69de29 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import java.text.SimpleDateFormat
-import java.util
-import java.util.{Date, List, Locale}
-
-import scala.collection.{immutable, mutable}
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
-import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.hadoop.mapreduce.task.{JobContextImpl,
TaskAttemptContextImpl}
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD,
PartitionedFile}
-import org.apache.spark.sql.util.CarbonException
-import org.apache.spark.sql.util.SparkSQLUtil.sessionState
-
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.constants.{CarbonCommonConstants,
CarbonLoadOptionConstants}
-import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil,
LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import
org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema,
CarbonLoadModel}
-import org.apache.carbondata.processing.util.{CarbonLoaderUtil,
DeleteLoadFolders, TableOptionConstant}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark.LOGGER
-import org.apache.carbondata.spark.load.ValidateUtil
-import org.apache.carbondata.spark.rdd.SerializableConfiguration
-
-/**
- * the util object of data loading
- */
-object DataLoadingUtil {
-
- val LOGGER: LogService =
LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- /**
- * get data loading options and initialise default value
- */
- def getDataLoadingOptions(
- carbonProperty: CarbonProperties,
- options: immutable.Map[String, String]): mutable.Map[String, String] = {
-val optionsFinal = scala.collection.mutable.Map[String, String]()
-optionsFinal.put("delimiter", options.getOrElse("delimiter", ","))
-optionsFinal.put("quotechar", options.getOrElse("quotechar", "\""))
-optionsFinal.put("fileheader", options.getOrElse("fileheader", ""))
-optionsFinal.put("commentchar", options.getOrElse("commentchar", "#"))
-optionsFinal.put("columndict", options.getOrElse("columndict", null))
-
-optionsFinal.put("escapechar",
- CarbonLoaderUtil.getEscapeChar(options.getOrElse("escapechar", "\\")))
-
-optionsFinal.put(
- "serialization_null_format",
- options.getOrElse("ser
[30/50] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
Pick up the no-sort fields in the row and pack them as bytes array and skip
parsing them during merge sort to reduce CPU consumption
This closes #1792
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8fcde02e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8fcde02e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8fcde02e
Branch: refs/heads/carbonstore-rebase5
Commit: 8fcde02eefbd9ff176ceed2c2a33fad4d48fb332
Parents: 0feaeb5
Author: xuchuanyin
Authored: Thu Feb 8 14:35:14 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 19:44:12 2018 +0800
--
.../carbondata/core/util/NonDictionaryUtil.java | 67 +--
.../presto/util/CarbonDataStoreCreator.scala| 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../loading/row/IntermediateSortTempRow.java| 117 +
.../loading/sort/SortStepRowHandler.java| 466 +++
.../loading/sort/SortStepRowUtil.java | 103
.../sort/unsafe/UnsafeCarbonRowPage.java| 331 ++---
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 +--
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 ---
.../UnsafeRowComparatorForNormalDims.java | 59 +++
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 ++
.../merger/UnsafeIntermediateFileMerger.java| 118 +
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 -
.../sort/sortdata/IntermediateFileMerger.java | 95 +---
.../IntermediateSortTempRowComparator.java | 73 +++
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94
.../sortdata/RowComparatorForNormalDims.java| 62 ---
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +---
.../sort/sortdata/SortTempFileChunkHolder.java | 174 ++-
.../sort/sortdata/TableFieldStat.java | 176 +++
28 files changed, 1186 insertions(+), 1294 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8fcde02e/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index d6ecfbc..fca1244 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,18 +82,26 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required Dimension from obj []
+ * Method to get the required dictionary Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static Integer getDimension(int index, Object[] row) {
-
-Integer[] dimensions = (Integer[])
row[WriteStepRowUtil.DICTIONARY_DIMENSION];
-
+ public static int getDictDimension(int index, Object[] row) {
+int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
return dimensions[index];
+ }
+ /**
+ * Method to get the required non-dictionary & complex from 3-parted row
+ * @param index
+ * @param row
+ * @return
+ */
+ public static byte[] getNoDictOrComplex(int index, Object[] row) {
+byte[][] nonDictArray = (byte[][])
row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+return nonDictArray[index];
}
/**
@@ -108,60 +116,11 @@ public class NonDictionaryUtil {
return measures[index];
}
- public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
-
-return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
- }
-
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][]
byteBufferArr,
Object[] measureArray) {
-
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
-
- /**
- * This method will extract the single dimension from the complete high card
dims byte[].+ *
- * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
- *
- * @param highCardArr
- * @param index
- * @param highCardinalityCount
- * @param outBuffer
- */
- public static v
[03/50] [abbrv] carbondata git commit: [CARBONDATA-1827] S3 Carbon Implementation
[CARBONDATA-1827] S3 Carbon Implementation
1.Provide support for s3 in carbondata.
2.Added S3Example to create carbon table on s3.
3.Added S3CSVExample to load carbon table using csv from s3.
This closes #1805
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4628d2ec
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4628d2ec
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4628d2ec
Branch: refs/heads/carbonstore-rebase5
Commit: 4628d2ec7608eabea5970c2106e6b4e785b5096b
Parents: 39d6748
Author: SangeetaGulia
Authored: Thu Sep 21 14:56:26 2017 +0530
Committer: Jacky Li
Committed: Sun Mar 4 18:15:05 2018 +0800
--
.../core/constants/CarbonCommonConstants.java | 21 +++
.../filesystem/AbstractDFSCarbonFile.java | 20 ++-
.../datastore/filesystem/HDFSCarbonFile.java| 5 +-
.../core/datastore/impl/FileFactory.java| 11 +-
.../core/locks/CarbonLockFactory.java | 28 ++--
.../carbondata/core/locks/S3FileLock.java | 111 +
.../carbondata/core/util/CarbonProperties.java | 3 +-
.../filesystem/HDFSCarbonFileTest.java | 8 +-
examples/spark2/pom.xml | 5 +
examples/spark2/src/main/resources/data1.csv| 11 ++
.../carbondata/examples/S3CsvExample.scala | 99 +++
.../apache/carbondata/examples/S3Example.scala | 164 +++
.../spark/rdd/NewCarbonDataLoadRDD.scala| 42 -
integration/spark2/pom.xml | 43 +
.../spark/rdd/CarbonDataRDDFactory.scala| 3 +-
.../org/apache/spark/sql/CarbonSession.scala| 3 +
16 files changed, 554 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4628d2ec/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b2a3375..af3ed99 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -167,6 +167,22 @@ public final class CarbonCommonConstants {
public static final String S3N_PREFIX = "s3n://";
public static final String S3A_PREFIX = "s3a://";
+ /**
+ * Access Key for s3n
+ */
+ public static final String S3N_ACCESS_KEY = "fs.s3n.awsAccessKeyId";
+ /**
+ * Secret Key for s3n
+ */
+ public static final String S3N_SECRET_KEY = "fs.s3n.awsSecretAccessKey";
+ /**
+ * Access Key for s3
+ */
+ public static final String S3_ACCESS_KEY = "fs.s3.awsAccessKeyId";
+ /**
+ * Secret Key for s3
+ */
+ public static final String S3_SECRET_KEY = "fs.s3.awsSecretAccessKey";
/**
* FS_DEFAULT_FS
@@ -937,6 +953,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_LOCK_TYPE_HDFS = "HDFSLOCK";
/**
+ * S3LOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_S3 = "S3LOCK";
+
+ /**
* Invalid filter member log string
*/
public static final String FILTER_INVALID_MEMBER =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4628d2ec/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 68eaa21..fd5dc40 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
/**
* LOGGER
*/
@@ -262,18 +262,28 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
@Override public DataOutputStream getDataOutputStream(String path,
FileFactory.FileType fileType,
int bufferSize, boolean append) throws IOException {
Path pt = new Path(path);
-FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());
FSDataOutputStream stream = null;
if (append) {
// appen
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix bugs in tests in windows env
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Fix bugs in tests that will cause failure under windows env
This closes #1994
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87bda960
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87bda960
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87bda960
Branch: refs/heads/carbonstore-rebase5
Commit: 87bda9601ed23871454640432025858799bff1f6
Parents: 49d06c2
Author: xuchuanyin
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:08:31 2018 +0800
--
.../carbondata/core/locks/LocalFileLock.java| 30 +++-
.../store/impl/DFSFileReaderImplUnitTest.java | 11 ---
.../store/impl/FileFactoryImplUnitTest.java | 28 +-
.../filesystem/HDFSCarbonFileTest.java | 3 +-
.../filesystem/LocalCarbonFileTest.java | 20 +
.../BooleanDataTypesInsertTest.scala| 5 ++--
.../carbondata/lcm/locks/LocalFileLockTest.java | 2 +-
.../loading/csvinput/CSVInputFormatTest.java| 1 +
8 files changed, 64 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
package org.apache.carbondata.core.locks;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
private String location;
/**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
* channel is the FileChannel of the lock file.
*/
private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
FileFactory.createNewLockFile(lockFilePath,
FileFactory.getFileType(location));
}
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
+ channel = FileChannel.open(Paths.get(lockFilePath),
StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND);
try {
fileLock = channel.tryLock();
} catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
} catch (IOException e) {
status = false;
} finally {
- if (null != fileOutputStream) {
-try {
- fileOutputStream.close();
-} catch (IOException e) {
- LOGGER.error(e.getMessage());
+ CarbonUtil.closeStreams(channel);
+
+ // deleting the lock file after releasing the lock.
+ if (null != lockFilePath) {
+CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+FileFactory.getFileType(lockFilePath));
+if (!lockFile.exists() || lockFile.delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+} else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/87bda960/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
---
a/core/src/test/java/org
[06/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index f4450e3..5f8d199 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import mockit.Mock;
import mockit.MockUp;
@@ -267,8 +267,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, 1);
}
@@ -276,8 +276,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValueToTarget() {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, -1);
}
@@ -285,8 +285,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 3);
}
@@ -302,8 +302,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValueToTarget() {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 5);
}
@@ -525,23 +525,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDictionaryEncodingArray() {
-QueryDimension column1 = new QueryDimension("Column1");
-QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List encoding = new ArrayList<>();
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
-column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ProjectionDimension
+column1 = new ProjectionDimension(new CarbonDimension(column1Schema,
1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
-column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ProjectionDimension
+column2 = new ProjectionDimension(new CarbonDimension(column2Schema,
1, 1, 1, 1));
-Que
[50/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after merging master
[REBASE] Solve conflict after merging master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7f92fde4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7f92fde4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7f92fde4
Branch: refs/heads/carbonstore-rebase5
Commit: 7f92fde4953a67470661e8ece7cb65bb0c9624bc
Parents: 87bda96
Author: Jacky Li
Authored: Tue Feb 27 11:26:30 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:23:37 2018 +0800
--
.../carbondata/core/datamap/dev/DataMap.java| 6 -
.../core/datamap/dev/DataMapFactory.java| 2 +-
.../exception/ConcurrentOperationException.java | 16 +-
.../core/metadata/PartitionMapFileStore.java| 0
.../statusmanager/SegmentStatusManager.java | 10 +-
.../SegmentUpdateStatusManager.java | 1 -
datamap/examples/pom.xml| 145 +++--
.../datamap/examples/MinMaxDataWriter.java | 1 -
examples/flink/pom.xml | 4 +-
.../carbondata/examples/FlinkExample.scala | 10 +-
.../CarbonStreamSparkStreamingExample.scala | 1 -
.../hadoop/api/CarbonTableInputFormat.java | 5 +-
.../TestInsertAndOtherCommandConcurrent.scala | 2 +-
.../StandardPartitionGlobalSortTestCase.scala | 2 +-
.../exception/ProcessMetaDataException.java | 2 +
.../org/apache/carbondata/api/CarbonStore.scala | 6 +-
.../carbondata/spark/load/CsvRDDHelper.scala| 157 +++
.../load/DataLoadProcessBuilderOnSpark.scala| 3 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 2 -
.../command/carbonTableSchemaCommon.scala | 6 +-
.../CarbonAlterTableCompactionCommand.scala | 3 +-
.../management/CarbonCleanFilesCommand.scala| 2 +-
.../CarbonDeleteLoadByIdCommand.scala | 2 +-
.../CarbonDeleteLoadByLoadDateCommand.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 28 ++--
.../CarbonProjectForDeleteCommand.scala | 2 +-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 3 -
.../vectorreader/AddColumnTestCases.scala | 1 +
.../datamap/DataMapWriterListener.java | 3 +-
.../loading/model/CarbonLoadModelBuilder.java | 34 +++-
.../processing/loading/model/LoadOption.java| 15 +-
.../processing/merger/CarbonDataMergerUtil.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 3 +-
.../processing/util/CarbonLoaderUtil.java | 8 +
store/sdk/pom.xml | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java| 8 +-
40 files changed, 336 insertions(+), 172 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7f92fde4/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 02db8af..dd5507c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -38,9 +38,6 @@ public interface DataMap {
/**
* Prune the datamap with filter expression and partition information. It
returns the list of
* blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
*/
List prune(FilterResolverIntf filterExp, SegmentProperties
segmentProperties,
List partitions);
@@ -48,9 +45,6 @@ public interface DataMap {
// TODO Move this method to Abstract class
/**
* Validate whether the current segment needs to be fetching the required
data
- *
- * @param filterExp
- * @return
*/
boolean isScanRequired(FilterResolverIntf filterExp);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7f92fde4/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index 50ac279..d8a467f 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,8 +21,8 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apa
[21/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b9125b6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
--
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 31a08fc..9afb890 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -26,7 +26,7 @@ import
org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.statusmanager.{SegmentStatus,
SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.CacheClient
import org.apache.spark.sql.test.util.QueryTest
@@ -112,11 +112,9 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest
with BeforeAndAfterAll
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
"ignoremajor"
)
-val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
-val carbontablePath =
CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- .getMetadataDirectoryPath
-val segs = SegmentStatusManager.readLoadMetadata(carbontablePath)
+val carbonTablePath = carbonTable.getMetadataPath
+val segs = SegmentStatusManager.readLoadMetadata(carbonTablePath)
// status should remain as compacted.
assertResult(SegmentStatus.COMPACTED)(segs(3).getSegmentStatus)
@@ -134,9 +132,7 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest
with BeforeAndAfterAll
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
"ignoremajor"
)
-val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
-val carbontablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier).getMetadataDirectoryPath
+val carbontablePath = carbonTable.getMetadataPath
val segs = SegmentStatusManager.readLoadMetadata(carbontablePath)
// status should remain as compacted for segment 2.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b9125b6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
--
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index 42ac4df..68a3058 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.test.util.QueryTest
import
org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
var filePath: String = _
@@ -193,8 +193,7 @@ class TestBatchSortDataLoad extends QueryTest with
BeforeAndAfterAll {
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
tableName
)
-val carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
-val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
+val segmentDir = carbonTable.getSemgentPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b9125b6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
--
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index db0a62c..b9d8e12 100644
---
a
[04/50] [abbrv] carbondata git commit: [CARBONDATA-1968] Add external table support
[CARBONDATA-1968] Add external table support
This PR adds support for creating external table with existing carbondata
files, using Hive syntax.
CREATE EXTERNAL TABLE tableName STORED BY 'carbondata' LOCATION 'path'
This closes #1749
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/39d6748d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/39d6748d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/39d6748d
Branch: refs/heads/carbonstore-rebase5
Commit: 39d6748dbc91200cb2e37afd7bb82af2bdea794f
Parents: c770ff6
Author: Jacky Li
Authored: Tue Jan 2 23:46:14 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 18:15:05 2018 +0800
--
.../core/metadata/schema/table/CarbonTable.java | 9 ++
.../createTable/TestCreateExternalTable.scala | 91
.../TestDataWithDicExcludeAndInclude.scala | 10 ---
.../command/table/CarbonDropTableCommand.scala | 5 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 64 +-
5 files changed, 147 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d6748d/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 09ff440..6036569 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -785,6 +785,15 @@ public class CarbonTable implements Serializable {
&& !tableInfo.getParentRelationIdentifiers().isEmpty();
}
+ /**
+ * Return true if this is an external table (table with property
"_external"="true", this is
+ * an internal table property set during table creation)
+ */
+ public boolean isExternalTable() {
+String external =
tableInfo.getFactTable().getTableProperties().get("_external");
+return external != null && external.equalsIgnoreCase("true");
+ }
+
public long size() throws IOException {
Map dataIndexSize = CarbonUtil.calculateDataIndexSize(this);
Long dataSize =
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d6748d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
--
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
new file mode 100644
index 000..67370eb
--- /dev/null
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import java.io.File
+
+import org.apache.spark.sql.{AnalysisException, CarbonEnv}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
+
+ var originDataPath: String = _
+
+ override def beforeAll(): Unit = {
+sql("DROP TABLE IF EXISTS origin")
+// create carbon table and insert data
+sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'")
+sql("INSERT INTO origin select 100,'spark'")
+sql("INSERT INTO origin select 200,'hive'")
+originDataPath = s"$storeLocation/origin"
+ }
+
+ override def afterAll(): Unit = {
+sql("DROP TABLE IF EXISTS origin")
+ }
+
+ test("create external table with existing files") {
+assert(new
[12/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/55c4e438/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 9391ebd..44f7c07 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -16,20 +16,10 @@
*/
package org.apache.carbondata.core.scan.filter.executer;
-import java.util.ArrayList;
import java.util.BitSet;
-import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
@@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil;
public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
-
LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
* @param dimColResolvedFilterInfo
* @param segmentProperties
*/
@@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl
}
/**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk
dimensionColumnDataChunk,
- int numerOfRows) {
-BitSet bitSet = new BitSet(numerOfRows);
-bitSet.flip(0, numerOfRows);
-try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
-byte[] filterVal = filterValues[i];
-for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new
byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId,
keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData)
== 0) {
-bitSet.flip(rowId);
- }
-}
- }
-
-} catch (Exception e) {
- LOGGER.error(e);
-}
-
-return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
-int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
-KeyGenerator keyGenerator =
segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
-List mdKeyOrdinal = new ArrayList();
-mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(),
colGrpId));
-int[] maskByteRanges =
QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
-byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal,
keyGenerator);
-KeyStructureInfo restructureInfos = new KeyStructureInfo();
-restructureInfos.setKeyGenerator(keyGenerator);
-restructureInfos.setMaskByteRanges(maskByteRanges);
-restructureInfos.setMaxKey(maxKey);
-return restructureInfos;
- }
-
- /**
* Check if scan is required on given block based on min and max value
*/
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
@@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl
return bitSet;
}
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
-return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
-int[][] columnGroups = segmentProperties.getColumnGroups();
-int colGrpId = -1;
-for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
-colGrpId++;
-if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
-}
- }
-}
-return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int co
[24/50] [abbrv] carbondata git commit: [CARBONDATA-1480]Min Max Index Example for DataMap
[CARBONDATA-1480]Min Max Index Example for DataMap
Datamap Example. Implementation of Min Max Index through Datamap. And Using the
Index while prunning.
This closes #1359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3a0a9e64
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3a0a9e64
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3a0a9e64
Branch: refs/heads/carbonstore-rebase5
Commit: 3a0a9e64a75bb4f1f4644775708fc55c256a0d8d
Parents: f8841d8
Author: sounakr
Authored: Thu Sep 28 16:21:05 2017 +0530
Committer: Jacky Li
Committed: Sun Mar 4 19:31:39 2018 +0800
--
.../core/datamap/DataMapStoreManager.java | 16 +-
.../carbondata/core/datamap/TableDataMap.java | 18 +-
.../carbondata/core/datamap/dev/DataMap.java| 11 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../indexstore/SegmentPropertiesFetcher.java| 36 +++
.../blockletindex/BlockletDataMap.java | 5 +-
.../blockletindex/BlockletDataMapFactory.java | 32 ++-
datamap/examples/pom.xml| 111 ++
.../datamap/examples/BlockletMinMax.java| 41
.../datamap/examples/MinMaxDataMap.java | 143
.../datamap/examples/MinMaxDataMapFactory.java | 114 ++
.../datamap/examples/MinMaxDataWriter.java | 221 +++
.../examples/MinMaxIndexBlockDetails.java | 77 +++
.../MinMaxDataMapExample.scala | 77 +++
.../testsuite/datamap/DataMapWriterSuite.scala | 2 +-
pom.xml | 2 +
.../datamap/DataMapWriterListener.java | 4 +-
.../store/writer/AbstractFactDataWriter.java| 7 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 3 +
19 files changed, 894 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a0a9e64/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index d30483a..90e5fff 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -103,7 +104,7 @@ public final class DataMapStoreManager {
tableDataMaps = new ArrayList<>();
}
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
-if (dataMap != null) {
+if (dataMap != null &&
dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
throw new RuntimeException("Already datamap exists in that path with
type " + dataMapName);
}
@@ -113,12 +114,15 @@ public final class DataMapStoreManager {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory,
blockletDetailsFetcher);
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher)
blockletDetailsFetcher;
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory,
blockletDetailsFetcher,
+ segmentPropertiesFetcher);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -128,11 +132,11 @@ public final class DataMapStoreManager {
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName,
- List tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List
tableDataMaps) {
TableDataMap dataMap = null;
-for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName)) {
+for (TableDataMa
[45/50] [abbrv] carbondata git commit: [CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data loading
http://git-wip-us.apache.org/repos/asf/carbondata/blob/49d06c20/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
--
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
deleted file mode 100644
index f605b22..000
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort.impl;
-
-import java.io.File;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.schema.BucketingInfo;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.processing.loading.DataField;
-import
org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
-import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
-import org.apache.carbondata.processing.loading.sort.AbstractMergeSorter;
-import
org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows;
-import
org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
-import
org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeSingleThreadFinalSortFilesMerger;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-
-/**
- * It parallely reads data from array of iterates and do merge sort.
- * First it sorts the data and write to temp files. These temp files will be
merge sorted to get
- * final merge sort result.
- * This step is specifically for bucketing, it sorts each bucket data
separately and write to
- * temp files.
- */
-public class UnsafeParallelReadMergeSorterWithBucketingImpl extends
AbstractMergeSorter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(
-
UnsafeParallelReadMergeSorterWithBucketingImpl.class.getName());
-
- private SortParameters sortParameters;
-
- private BucketingInfo bucketingInfo;
-
- public UnsafeParallelReadMergeSorterWithBucketingImpl(DataField[]
inputDataFields,
- BucketingInfo bucketingInfo) {
-this.bucketingInfo = bucketingInfo;
- }
-
- @Override public void initialize(SortParameters sortParameters) {
-this.sortParameters = sortParameters;
- }
-
- @Override public Iterator[] sort(Iterator[]
iterators)
- throws CarbonDataLoadingException {
-UnsafeSortDataRows[] sortDataRows = new
UnsafeSortDataRows[bucketingInfo.getNumberOfBuckets()];
-UnsafeIntermediateMerger[] intermediateFileMergers =
-new UnsafeIntermediateMerger[sortDataRows.length];
-int inMemoryChunkSizeInMB =
CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
-inMemoryChunkSizeInMB = inMemoryChunkSizeInMB /
bucketingInfo.getNumberOfBuckets();
-if (inMemoryChunkSizeInMB < 5) {
- inMemoryChunkSizeInMB = 5;
-}
-try {
- for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
-SortParameters parameters = sortParameters.getCopy();
-parameters.setPartitionID(i + "");
-setTempLocation(parameters);
-intermediateFileMergers[
[26/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/828ae5ec/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
--
diff --git
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index b196d0d..266c107 100644
---
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.datamap.examples;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,90 +24,94 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.events.ChangeEvent;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import
org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import
org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxDataMapFactory implements DataMapFactory {
+public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, String
dataMapName) {
this.identifier = identifier;
}
/**
* createWriter will return the MinMaxDataWriter.
- * @param segmentId
+ *
+ * @param segment
* @return
*/
- @Override
- public DataMapWriter createWriter(String segmentId) {
-return new MinMaxDataWriter();
+ @Override public AbstractDataMapWriter createWriter(Segment segment) {
+return new MinMaxDataWriter(identifier, segment.getSegmentNo(),
+CarbonTablePath.getSegmentPath(identifier.getTablePath(),
segment.getSegmentNo()));
}
/**
* getDataMaps Factory method Initializes the Min Max Data Map and returns.
- * @param segmentId
+ *
+ * @param segment
* @return
* @throws IOException
*/
- @Override public List getDataMaps(String segmentId) throws
IOException {
-List dataMapList = new ArrayList<>();
+ @Override public List getDataMaps(Segment
segment)
+ throws IOException {
+List dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxDataMap.
MinMaxDataMap dataMap = new MinMaxDataMap();
try {
- dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" +
segmentId + File.separator);
+ dataMap.init(new DataMapModel(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(),
segment.getSegmentNo(;
} catch (MemoryException ex) {
-
+ throw new IOException(ex);
}
dataMapList.add(dataMap);
return dataMapList;
}
/**
- *
- * @param segmentId
+ * @param segment
* @return
*/
- @Override public List toDistributable(String
segmentId) {
+ @Override public List toDistributable(Segment segment)
{
return null;
}
/**
* Clear the DataMap.
- * @param segmentId
+ *
+ * @param segment
*/
- @Override public void clear(String segmentId) {
+ @Override public void clear(Segment segment) {
}
/**
* Clearing the data map.
*/
- @Override
- public void clear() {
+ @Override public void clear() {
}
- @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ @Override public List
getDataMaps(DataMapDistributable distributable)
+ throws IOException {
return null;
}
- @Override
- public void fireEvent(ChangeEvent event) {
+ @Override public void fireEvent(Event event) {
}
- @Override
- public DataMapMeta getMeta() {
-return new DataMapMeta(new ArrayList(Arrays.asList("c2")),
FilterType.EQUALTO);
+ @Override public DataMapMeta getMeta() {
+return new DataMapMeta(new ArrayList(Arr
[32/50] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/111bb5c4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/111bb5c4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/111bb5c4
Branch: refs/heads/carbonstore-rebase5
Commit: 111bb5c41946d72604964658ec8562f7722dec14
Parents: ef81248
Author: xuchuanyin
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 19:54:03 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/block/TableBlockInfo.java| 29 ++
.../carbondata/core/util/CarbonProperties.java | 11 +
docs/useful-tips-on-carbondata.md | 1 +
.../CarbonIndexFileMergeTestCase.scala | 4 -
.../StandardPartitionTableLoadingTestCase.scala | 2 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 ++
.../processing/util/CarbonLoaderUtil.java | 494 ---
.../processing/util/CarbonLoaderUtilTest.java | 125 +
12 files changed, 552 insertions(+), 188 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111bb5c4/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index bcfeba0..a6bf60f 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,4 +114,14 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
+ /**
+ * enable block size based block allocation while loading data. By default,
carbondata assigns
+ * blocks to node based on block number. If this option is set to `true`,
carbondata will
+ * consider block size first and make sure that all the nodes will process
almost equal size of
+ * data. This option is especially useful when you encounter skewed data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
+ = "carbon.load.skewedDataOptimization.enabled";
+ public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111bb5c4/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a7bfdba..c0cebe0 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -98,6 +100,20 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
+ /**
+ * comparator to sort by block size in descending order.
+ * Since each line is not exactly the same, the size of a InputSplit may
differs,
+ * so we allow some deviation for these splits.
+ */
+ public static final Comparator DATA_SIZE_DESC_COMPARATOR =
+ new Comparator() {
+@Override public int compare(Distributable o1, Distributable o2) {
+ long diff =
+ ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
+ return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
+}
+ };
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] del
[01/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after rebasing master [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/carbonstore-rebase5 7f92fde49 -> 8104735fd (forced update)
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bd40a0d7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bd40a0d7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bd40a0d7
Branch: refs/heads/carbonstore-rebase5
Commit: bd40a0d73d2a7086caaa6773a2c6a1a45e24334c
Parents: 9086a1b
Author: Jacky Li
Authored: Thu Feb 1 00:25:31 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:30:31 2018 +0800
--
.../hadoop/util/CarbonInputFormatUtil.java | 20 +++
.../spark/rdd/NewCarbonDataLoadRDD.scala| 21 ++--
.../org/apache/spark/sql/CarbonSession.scala| 5 ++---
3 files changed, 24 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd40a0d7/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
--
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 514428b..056c27b 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -22,6 +22,8 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Locale;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -39,6 +41,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
@@ -159,4 +162,21 @@ public class CarbonInputFormatUtil {
String jobtrackerID = createJobTrackerID(date);
return new JobID(jobtrackerID, batch);
}
+
+ public static void setS3Configurations(Configuration hadoopConf) {
+FileFactory.getConfiguration()
+.set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""));
+FileFactory.getConfiguration()
+.set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""));
+FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY,
+hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd40a0d7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 917fc88..e17824f 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,10 +41,10 @@ import
org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails,
SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties,
CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.hado
[07/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
--
diff --git
a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index f4450e3..5f8d199 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import
org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import mockit.Mock;
import mockit.MockUp;
@@ -267,8 +267,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, 1);
}
@@ -276,8 +276,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValueToTarget() {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk,
compareValues);
assertEquals(result, -1);
}
@@ -285,8 +285,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 3);
}
@@ -302,8 +302,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValueToTarget() {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
-FixedLengthDimensionDataChunk fixedLengthDataChunk =
-new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+FixedLengthDimensionColumnPage fixedLengthDataChunk =
+new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk,
compareValues, 5);
assertEquals(result, 5);
}
@@ -525,23 +525,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDictionaryEncodingArray() {
-QueryDimension column1 = new QueryDimension("Column1");
-QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List encoding = new ArrayList<>();
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
-column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ProjectionDimension
+column1 = new ProjectionDimension(new CarbonDimension(column1Schema,
1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
-column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ProjectionDimension
+column2 = new ProjectionDimension(new CarbonDimension(column2Schema,
1, 1, 1, 1));
-Que
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 8c8d08f..a689d8e 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -124,22 +124,22 @@ public class UnsafeFixedLengthDimensionDataChunkStore
/**
* to compare the two byte array
*
- * @param indexindex of first byte array
+ * @param rowIdindex of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// based on index we need to calculate the actual position in memory block
-index = index * columnValueSize;
+rowId = rowId * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
compareResult = (CarbonUnsafe.getUnsafe()
- .getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + index)
+ .getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + rowId)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
break;
}
- index++;
+ rowId++;
}
return compareResult;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 36b2bd8..e1eb378 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -189,11 +189,11 @@ public class UnsafeVariableLengthDimesionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -201,13 +201,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// else subtract the current row offset
// with complete data length get the offset of set of data
int currentDataOffset =
CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
-dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((long)index
+dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((long) rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
-if (index < numberOfRows - 1) {
+if (rowId < numberOfRows - 1) {
int OffsetOfNextdata =
CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((index + 1)
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets +
((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
index 74d268a..e2a4161 100644
---
a/core/src/main/java/org/apache/
[08/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 94a041a..b74c279 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -378,7 +378,6 @@ public abstract class AbstractDataFileFooterConverter {
cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
}
info.setColumnCardinality(cardinality);
-info.setNumberOfColumns(segmentInfo.getNum_cols());
return info;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 52305bd..0cc783e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -48,10 +48,10 @@ import
org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
@@ -82,7 +82,7 @@ import
org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
@@ -248,16 +248,13 @@ public final class CarbonUtil {
public static ColumnGroupModel getColGroupModel(int[][] columnGroups) {
int[] columnSplit = new int[columnGroups.length];
int noOfColumnStore = columnSplit.length;
-boolean[] columnarStore = new boolean[noOfColumnStore];
for (int i = 0; i < columnGroups.length; i++) {
columnSplit[i] = columnGroups[i].length;
- columnarStore[i] = columnGroups[i].length <= 1;
}
ColumnGroupModel colGroupModel = new ColumnGroupModel();
colGroupModel.setNoOfColumnStore(noOfColumnStore);
colGroupModel.setColumnSplit(columnSplit);
-colGroupModel.setColumnarStore(columnarStore);
colGroupModel.setColumnGroup(columnGroups);
return colGroupModel;
}
@@ -418,7 +415,7 @@ public final class CarbonUtil {
}
}
- public static int getFirstIndexUsingBinarySearch(DimensionColumnDataChunk
dimColumnDataChunk,
+ public static int getFirstIndexUsingBinarySearch(DimensionColumnPage
dimColumnDataChunk,
int low, int high, byte[] compareValue, boolean matchUpLimit) {
int cmpResult = 0;
while (high >= low) {
@@ -457,7 +454,7 @@ public final class CarbonUtil {
* @return the compareValue's range index in the dimColumnDataChunk
*/
public static int[] getRangeIndexUsingBinarySearch(
- DimensionColumnDataChunk dimColumnDataChunk, int low, int high, byte[]
compareValue) {
+ DimensionColumnPage dimColumnDataChunk, int low, int high, byte[]
compareValue) {
int[] rangeIndex = new int[2];
int cmpResult = 0;
@@ -551,7 +548,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextLesserValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue) {
while (currentIndex - 1 >= 0
&& dimColumnDataChunk.compareTo(currentIndex - 1, compareValue) >= 0) {
--currentIndex;
@@ -571,7 +568,7 @@ public
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f06824e9/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 9799ac2..b7b5e43 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -22,16 +22,14 @@ import
org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.hadoop.fs.Path;
-
/**
* Helps to get Table content paths.
*/
-public class CarbonTablePath extends Path {
+public class CarbonTablePath {
private static final String METADATA_DIR = "Metadata";
private static final String DICTIONARY_EXT = ".dict";
@@ -54,19 +52,10 @@ public class CarbonTablePath extends Path {
private static final String STREAMING_LOG_DIR = "log";
private static final String STREAMING_CHECKPOINT_DIR = "checkpoint";
- private String tablePath;
- private CarbonTableIdentifier carbonTableIdentifier;
-
/**
- * structure CarbonTablePath object to manage table paths
- *
- * @param carbonTableIdentifier identifier of carbon table that the segment
belong to
- * @param tablePathString the store path of the segment
+ * This class provides static utility only.
*/
- public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String
tablePathString) {
-super(tablePathString);
-this.carbonTableIdentifier = carbonTableIdentifier;
-this.tablePath = tablePathString;
+ private CarbonTablePath() {
}
/**
@@ -130,40 +119,21 @@ public class CarbonTablePath extends Path {
}
/**
- * gets table path
+ * Return absolute path of dictionary file
*/
- public String getPath() {
-return tablePath;
+ public static String getDictionaryFilePath(String tablePath, String
columnId) {
+return getMetadataPath(tablePath) + File.separator +
getDictionaryFileName(columnId);
}
/**
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of dictionary file
*/
- public String getDictionaryFilePath(String columnId) {
-return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
- }
-
- /**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
- */
- public String getDictionaryFilePath(String dictionaryPath, String columnId) {
+ public static String getExternalDictionaryFilePath(String dictionaryPath,
String columnId) {
return dictionaryPath + File.separator + getDictionaryFileName(columnId);
}
/**
- * This method will return the metadata directory location for a table
- *
- * @return
- */
- public String getMetadataDirectoryPath() {
-return getMetaDataDir();
- }
-
- /**
- * Return metadata path based on `tablePath`
+ * Return metadata path
*/
public static String getMetadataPath(String tablePath) {
return tablePath + File.separator + METADATA_DIR;
@@ -184,67 +154,42 @@ public class CarbonTablePath extends Path {
}
/**
- * @param columnId unique column identifier
- * @return absolute path of dictionary meta file
+ * Return absolute path of dictionary meta file
*/
- public String getDictionaryMetaFilePath(String columnId) {
-return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
- }
-
- /**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
- */
- public String getDictionaryMetaFilePath(String dictionaryPath, String
columnId) {
+ public static String getExternalDictionaryMetaFilePath(String
dictionaryPath, String columnId) {
return dictionaryPath + File.separator + columnId + DICTIONARY_META_EXT;
}
/**
- * @param columnId unique column identifier
- * @return absolute path of sort index file
+ * Return absolute path of dictionary meta file
*/
- public String getSortIndexFilePath(String columnId) {
-return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+ public static String getDictionaryMetaFilePath(String tablePath, String
columnId) {
+return getMetadataPath(tablePath) + File.separator + columnId +
DICTIONARY_META_EXT;
}
/**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary
[34/50] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8fe8ab4c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8fe8ab4c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8fe8ab4c
Branch: refs/heads/carbonstore-rebase5
Commit: 8fe8ab4c078de0ccd218f8ba41352896aebd5202
Parents: 28b5720
Author: xuchuanyin
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:32:13 2018 +0800
--
.../constants/CarbonLoadOptionConstants.java| 10 +
.../core/datastore/block/TableBlockInfo.java| 29 ++
.../carbondata/core/util/CarbonProperties.java | 11 +
docs/useful-tips-on-carbondata.md | 1 +
.../spark/rdd/NewCarbonDataLoadRDD.scala| 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala| 18 +-
.../merger/NodeMultiBlockRelation.java | 40 ++
.../processing/util/CarbonLoaderUtil.java | 480 ---
.../processing/util/CarbonLoaderUtilTest.java | 125 +
10 files changed, 545 insertions(+), 175 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8fe8ab4c/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index bcfeba0..a6bf60f 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,4 +114,14 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 1000;
+ /**
+ * enable block size based block allocation while loading data. By default,
carbondata assigns
+ * blocks to node based on block number. If this option is set to `true`,
carbondata will
+ * consider block size first and make sure that all the nodes will process
almost equal size of
+ * data. This option is especially useful when you encounter skewed data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
+ = "carbon.load.skewedDataOptimization.enabled";
+ public static final String
ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8fe8ab4c/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a7bfdba..c0cebe0 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -98,6 +100,20 @@ public class TableBlockInfo implements Distributable,
Serializable {
private String dataMapWriterPath;
+ /**
+ * comparator to sort by block size in descending order.
+ * Since each line is not exactly the same, the size of a InputSplit may
differs,
+ * so we allow some deviation for these splits.
+ */
+ public static final Comparator DATA_SIZE_DESC_COMPARATOR =
+ new Comparator() {
+@Override public int compare(Distributable o1, Distributable o2) {
+ long diff =
+ ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo)
o2).getBlockLength();
+ return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
+}
+ };
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -434,4 +450,17 @@ public class TableBlockInfo implements Distributable,
Serializable {
[13/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 9391ebd..44f7c07 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -16,20 +16,10 @@
*/
package org.apache.carbondata.core.scan.filter.executer;
-import java.util.ArrayList;
import java.util.BitSet;
-import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
@@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil;
public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
-
LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
* @param dimColResolvedFilterInfo
* @param segmentProperties
*/
@@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl
}
/**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk
dimensionColumnDataChunk,
- int numerOfRows) {
-BitSet bitSet = new BitSet(numerOfRows);
-bitSet.flip(0, numerOfRows);
-try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
-byte[] filterVal = filterValues[i];
-for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new
byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId,
keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData)
== 0) {
-bitSet.flip(rowId);
- }
-}
- }
-
-} catch (Exception e) {
- LOGGER.error(e);
-}
-
-return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
-int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
-KeyGenerator keyGenerator =
segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
-List mdKeyOrdinal = new ArrayList();
-mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(),
colGrpId));
-int[] maskByteRanges =
QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
-byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal,
keyGenerator);
-KeyStructureInfo restructureInfos = new KeyStructureInfo();
-restructureInfos.setKeyGenerator(keyGenerator);
-restructureInfos.setMaskByteRanges(maskByteRanges);
-restructureInfos.setMaxKey(maxKey);
-return restructureInfos;
- }
-
- /**
* Check if scan is required on given block based on min and max value
*/
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
@@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends
ExcludeFilterExecuterImpl
return bitSet;
}
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
-return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
-int[][] columnGroups = segmentProperties.getColumnGroups();
-int colGrpId = -1;
-for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
-colGrpId++;
-if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
-}
- }
-}
-return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int co
[33/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap. FG Query flow as follows. 1.The user can add FG datamap to any table and implement there interfaces. 2. Any filter query which hits the table with datamap will call prune method of FGdatamap. 3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well. 4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits. 5. Based on the splits scanrdd schedule the tasks. 6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it. This closes #1471 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bb5bb00a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bb5bb00a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bb5bb00a Branch: refs/heads/carbonstore-rebase5 Commit: bb5bb00af982831ea83c73a2c437aa4aea8a5422 Parents: f9d15a2 Author: ravipesala Authored: Wed Nov 15 19:48:40 2017 +0530 Committer: Jacky Li Committed: Sun Mar 4 20:32:12 2018 +0800 -- .../carbondata/core/datamap/DataMapMeta.java| 8 +- .../core/datamap/DataMapStoreManager.java | 30 +- .../carbondata/core/datamap/DataMapType.java| 21 + .../carbondata/core/datamap/TableDataMap.java | 33 +- .../core/datamap/dev/AbstractDataMapWriter.java | 110 + .../core/datamap/dev/BlockletSerializer.java| 57 +++ .../carbondata/core/datamap/dev/DataMap.java| 5 +- .../core/datamap/dev/DataMapFactory.java| 14 +- .../core/datamap/dev/DataMapWriter.java | 57 --- .../cgdatamap/AbstractCoarseGrainDataMap.java | 24 + .../AbstractCoarseGrainDataMapFactory.java | 34 ++ .../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 + .../AbstractFineGrainDataMapFactory.java| 38 ++ .../carbondata/core/datastore/DataRefNode.java | 6 + .../core/datastore/block/TableBlockInfo.java| 10 + .../impl/btree/AbstractBTreeLeafNode.java | 5 + .../datastore/impl/btree/BTreeNonLeafNode.java | 5 + .../carbondata/core/indexstore/Blocklet.java| 30 +- .../indexstore/BlockletDataMapIndexStore.java | 6 - .../core/indexstore/BlockletDetailsFetcher.java | 9 + .../core/indexstore/ExtendedBlocklet.java | 17 + .../core/indexstore/FineGrainBlocklet.java | 120 + .../indexstore/SegmentPropertiesFetcher.java| 3 +- .../blockletindex/BlockletDataMap.java | 18 +- .../blockletindex/BlockletDataMapFactory.java | 66 ++- .../blockletindex/BlockletDataRefNode.java | 27 +- .../indexstore/blockletindex/IndexWrapper.java | 18 + .../core/indexstore/schema/FilterType.java | 24 - .../executer/ExcludeFilterExecuterImpl.java | 3 + .../executer/IncludeFilterExecuterImpl.java | 3 + .../scanner/impl/BlockletFilterScanner.java | 2 + .../apache/carbondata/core/util/CarbonUtil.java | 97 datamap/examples/pom.xml| 2 +- .../datamap/examples/MinMaxDataMap.java | 33 +- .../datamap/examples/MinMaxDataMapFactory.java | 67 +-- .../datamap/examples/MinMaxDataWriter.java | 36 +- .../examples/MinMaxIndexBlockDetails.java | 13 - .../carbondata/hadoop/CarbonInputSplit.java | 21 +- .../hadoop/api/CarbonTableInputFormat.java | 17 +- .../testsuite/datamap/CGDataMapTestCase.scala | 361 +++ .../testsuite/datamap/DataMapWriterSuite.scala | 43 +- .../testsuite/datamap/FGDataMapTestCase.scala | 440 +++ .../TestInsertAndOtherCommandConcurrent.scala | 21 +- .../carbondata/spark/rdd/CarbonScanRDD.scala| 7 +- .../TestStreamingTableOperation.scala | 5 +- .../datamap/DataMapWriterListener.java | 57 ++- .../store/CarbonFactDataHandlerModel.java | 10 +- .../store/writer/AbstractFactDataWriter.java| 128 +- 48 files changed, 1784 insertions(+), 401 deletions(-) -- http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb5bb00a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java -- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java index 7746acf..dd15ccb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java @@ -19,15 +19,15 @@ package org.apache.carbondata
[32/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap FineGrain implementation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bb5bb00a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
--
diff --git
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index b196d0d..266c107 100644
---
a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.datamap.examples;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,90 +24,94 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.events.ChangeEvent;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import
org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import
org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxDataMapFactory implements DataMapFactory {
+public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, String
dataMapName) {
this.identifier = identifier;
}
/**
* createWriter will return the MinMaxDataWriter.
- * @param segmentId
+ *
+ * @param segment
* @return
*/
- @Override
- public DataMapWriter createWriter(String segmentId) {
-return new MinMaxDataWriter();
+ @Override public AbstractDataMapWriter createWriter(Segment segment) {
+return new MinMaxDataWriter(identifier, segment.getSegmentNo(),
+CarbonTablePath.getSegmentPath(identifier.getTablePath(),
segment.getSegmentNo()));
}
/**
* getDataMaps Factory method Initializes the Min Max Data Map and returns.
- * @param segmentId
+ *
+ * @param segment
* @return
* @throws IOException
*/
- @Override public List getDataMaps(String segmentId) throws
IOException {
-List dataMapList = new ArrayList<>();
+ @Override public List getDataMaps(Segment
segment)
+ throws IOException {
+List dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxDataMap.
MinMaxDataMap dataMap = new MinMaxDataMap();
try {
- dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" +
segmentId + File.separator);
+ dataMap.init(new DataMapModel(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(),
segment.getSegmentNo(;
} catch (MemoryException ex) {
-
+ throw new IOException(ex);
}
dataMapList.add(dataMap);
return dataMapList;
}
/**
- *
- * @param segmentId
+ * @param segment
* @return
*/
- @Override public List toDistributable(String
segmentId) {
+ @Override public List toDistributable(Segment segment)
{
return null;
}
/**
* Clear the DataMap.
- * @param segmentId
+ *
+ * @param segment
*/
- @Override public void clear(String segmentId) {
+ @Override public void clear(Segment segment) {
}
/**
* Clearing the data map.
*/
- @Override
- public void clear() {
+ @Override public void clear() {
}
- @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ @Override public List
getDataMaps(DataMapDistributable distributable)
+ throws IOException {
return null;
}
- @Override
- public void fireEvent(ChangeEvent event) {
+ @Override public void fireEvent(Event event) {
}
- @Override
- public DataMapMeta getMeta() {
-return new DataMapMeta(new ArrayList(Arrays.asList("c2")),
FilterType.EQUALTO);
+ @Override public DataMapMeta getMeta() {
+return new DataMapMeta(new ArrayList(Arr
[26/50] [abbrv] carbondata git commit: [REBASE] resolve conflict after rebasing to master
[REBASE] resolve conflict after rebasing to master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6216294c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6216294c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6216294c
Branch: refs/heads/carbonstore-rebase5
Commit: 6216294c1e28c1db05e572f0aac3a991d345e085
Parents: 3fdd5d0
Author: Jacky Li
Authored: Tue Feb 27 08:51:25 2018 +0800
Committer: Jacky Li
Committed: Sun Mar 4 20:32:12 2018 +0800
--
.../core/datamap/dev/AbstractDataMapWriter.java | 5 ++--
.../core/datamap/dev/DataMapFactory.java| 2 +-
.../blockletindex/BlockletDataMapFactory.java | 2 +-
.../SegmentUpdateStatusManager.java | 9 +-
.../datamap/examples/MinMaxDataMapFactory.java | 5 ++--
.../datamap/examples/MinMaxDataWriter.java | 7 +++--
.../testsuite/datamap/CGDataMapTestCase.scala | 26
.../testsuite/datamap/DataMapWriterSuite.scala | 19 ++--
.../testsuite/datamap/FGDataMapTestCase.scala | 31 +---
.../iud/DeleteCarbonTableTestCase.scala | 2 +-
.../TestInsertAndOtherCommandConcurrent.scala | 14 +
.../StandardPartitionTableCleanTestCase.scala | 12
.../carbondata/spark/util/DataLoadingUtil.scala | 2 +-
.../datamap/DataMapWriterListener.java | 2 +-
.../processing/merger/CarbonDataMergerUtil.java | 8 +
.../merger/CompactionResultSortProcessor.java | 4 +--
.../merger/RowResultMergerProcessor.java| 5 ++--
.../partition/spliter/RowResultProcessor.java | 5 ++--
.../util/CarbonDataProcessorUtil.java | 4 +--
.../processing/util/CarbonLoaderUtil.java | 9 --
20 files changed, 73 insertions(+), 100 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6216294c/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
index bcc9bad..de6dcb1 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -35,10 +36,10 @@ public abstract class AbstractDataMapWriter {
protected String writeDirectoryPath;
- public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String
segmentId,
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, Segment
segment,
String writeDirectoryPath) {
this.identifier = identifier;
-this.segmentId = segmentId;
+this.segmentId = segment.getSegmentNo();
this.writeDirectoryPath = writeDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6216294c/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index df5670d..50ac279 100644
---
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -39,7 +39,7 @@ public interface DataMapFactory {
/**
* Return a new write for this datamap
*/
- AbstractDataMapWriter createWriter(Segment segment);
+ AbstractDataMapWriter createWriter(Segment segment, String
writeDirectoryPath);
/**
* Get the datamap for segmentid
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6216294c/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index efe2b71..ee849bd 100644
---
a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFacto
[12/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index de97e82..540607d 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import
org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import
org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -58,7 +58,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,20 +68,20 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List dimColEvaluatorInfoList;
- protected List msrColEvalutorInfoList;
+ List dimColEvaluatorInfoList;
+ List msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
protected SegmentProperties segmentProperties;
/**
* it has index at which given dimension is stored in file
*/
- protected int[] dimensionBlocksIndex;
+ int[] dimensionChunkIndex;
/**
* it has index at which given measure is stored in file
*/
- protected int[] measureBlocksIndex;
+ int[] measureChunkIndex;
private Map complexDimensionInfoMap;
@@ -89,18 +89,18 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
* flag to check whether the filter dimension is present in current block
list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean[] isDimensionPresentInCurrentBlock;
+ boolean[] isDimensionPresentInCurrentBlock;
/**
* flag to check whether the filter measure is present in current block list
of measures.
* Applicable for restructure scenarios
*/
- protected boolean[] isMeasurePresentInCurrentBlock;
+ boolean[] isMeasurePresentInCurrentBlock;
/**
* is dimension column data is natural sorted
*/
- protected boolean isNaturalSorted;
+ boolean isNaturalSorted;
/**
* date direct dictionary generator
@@ -124,10 +124,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new
boolean[dimColEvaluatorInfoList.size()];
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
- this.dimensionBlocksIndex = new int[]{0};
+ this.dimensionChunkIndex = new int[]{0};
}
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new
ArrayList(20);
@@ -136,10 +136,10 @@ public class RowLevelFilterExecuterImpl implements
FilterExecuter {
}
if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new
boolean[msrColEvalutorInfoList.size()];
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
- this.measureBlocksIndex = new int[] {0};
+ this.measureChunkInde
[17/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
deleted file mode 100644
index 6629d31..000
---
a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import
org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to fixed length dimension data chunk store
- */
-public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor
- *
- * @param dataChunkdata chunk
- * @param invertedIndexinverted index
- * @param invertedIndexReverse reverse inverted index
- * @param numberOfRows number of rows
- * @param columnValueSize size of each column value
- */
- public FixedLengthDimensionDataChunk(byte[] dataChunk, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows, int columnValueSize) {
-long totalSize = null != invertedIndex ?
-dataChunk.length + (2 * numberOfRows *
CarbonCommonConstants.INT_SIZE_IN_BYTE) :
-dataChunk.length;
-dataChunkStore = DimensionChunkStoreFactory.INSTANCE
-.getDimensionChunkStore(columnValueSize, null != invertedIndex,
numberOfRows, totalSize,
-DimensionStoreType.FIXEDLENGTH);
-dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param indexrow id of the chunk
- * @param keyStructureInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int index,
- KeyStructureInfo keyStructureInfo) {
-dataChunkStore.fillRow(index, data, offset);
-return dataChunkStore.getColumnValueSize();
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex,
int[] row,
- KeyStructureInfo restructuringInfo) {
-row[columnIndex] = dataChunkStore.getSurrogate(rowId);
-return columnIndex + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo,
int column,
- KeyStructureInfo restructuringInfo) {
-ColumnVectorInfo columnVectorInfo = vectorInfo[column];
-int offset = columnVectorInfo.offset;
-int vectorOffset = columnVectorInfo.vectorOffset;
-int len = columnVectorInfo.size + offset;
-CarbonColumnVector vector = columnVectorInfo.vector;
-for (int j = offset; j < len; j++) {
- int dict = dataChunkStore.getSurrogate(j);
- if (columnVectorInfo.directDictionaryGenerator == null) {
-vector.putInt(vectorOffset++, dict);
- } else {
-Object valueFromSurr
[09/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor query scan process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
index 553f85e..773fbd7 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
@@ -17,20 +17,15 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
public class PartitionSpliterRawResultIterator extends
CarbonIterator {
- private CarbonIterator iterator;
- private BatchResult batch;
+ private CarbonIterator iterator;
+ private RowBatch batch;
private int counter;
- private static final LogService LOGGER =
-
LogServiceFactory.getLogService(PartitionSpliterRawResultIterator.class.getName());
-
- public PartitionSpliterRawResultIterator(CarbonIterator
iterator) {
+ public PartitionSpliterRawResultIterator(CarbonIterator iterator) {
this.iterator = iterator;
}
@@ -65,7 +60,7 @@ public class PartitionSpliterRawResultIterator extends
CarbonIterator
* @param batch
* @return
*/
- private boolean checkBatchEnd(BatchResult batch) {
+ private boolean checkBatchEnd(RowBatch batch) {
return !(counter < batch.getSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
index 70d0958..1dd1595 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -37,7 +37,7 @@ public class RawResultIterator extends
CarbonIterator {
/**
* Iterator of the Batch raw result.
*/
- private CarbonIterator detailRawQueryResultIterator;
+ private CarbonIterator detailRawQueryResultIterator;
/**
* Counter to maintain the row counter.
@@ -55,9 +55,9 @@ public class RawResultIterator extends
CarbonIterator {
/**
* batch of the result.
*/
- private BatchResult batch;
+ private RowBatch batch;
- public RawResultIterator(CarbonIterator
detailRawQueryResultIterator,
+ public RawResultIterator(CarbonIterator
detailRawQueryResultIterator,
SegmentProperties sourceSegProperties, SegmentProperties
destinationSegProperties) {
this.detailRawQueryResultIterator = detailRawQueryResultIterator;
this.sourceSegProperties = sourceSegProperties;
@@ -155,7 +155,7 @@ public class RawResultIterator extends
CarbonIterator {
* @param batch
* @return
*/
- private boolean checkIfBatchIsProcessedCompletely(BatchResult batch) {
+ private boolean checkIfBatchIsProcessedCompletely(RowBatch batch) {
if (counter < batch.getSize()) {
return false;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92c9f224/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
index cc9710e..c7cb00d 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -35,10 +35,12 @@ public class VectorDetailQueryResultIterator ext
[29/50] [abbrv] carbondata git commit: [CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to executor for s3 implementation in cluster mode.
[CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to
executor for s3 implementation in cluster mode.
Problem : hadoopconf was not getting propagated from driver to the executor
that's why load was failing to the distributed environment.
Solution: Setting the Hadoop conf in base class CarbonRDD
How to verify this PR :
Execute the load in the cluster mode It should be a success using location s3.
This closes #1860
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dfbdf3db
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dfbdf3db
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dfbdf3db
Branch: refs/heads/carbonstore-rebase5
Commit: dfbdf3db00cbb488e49d3125b4ec93ff9e0dc9b2
Parents: f06824e
Author: Jatin
Authored: Thu Jan 25 16:53:00 2018 +0530
Committer: Jacky Li
Committed: Sun Mar 4 20:32:12 2018 +0800
--
.../spark/rdd/AlterTableAddColumnRDD.scala | 2 +-
.../spark/rdd/AlterTableDropColumnRDD.scala | 2 +-
.../spark/rdd/CarbonCleanFilesRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadRDD.scala | 2 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 2 +-
.../spark/rdd/CarbonDropTableRDD.scala | 2 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +-
.../spark/rdd/CarbonMergeFilesRDD.scala | 0
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
.../apache/carbondata/spark/rdd/CarbonRDD.scala | 32 ++--
.../spark/rdd/NewCarbonDataLoadRDD.scala| 2 +-
.../carbondata/spark/rdd/SparkDataMapJob.scala | 2 +-
.../apache/spark/rdd/DataLoadCoalescedRDD.scala | 3 +-
.../apache/spark/rdd/UpdateCoalescedRDD.scala | 2 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
16 files changed, 45 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfbdf3db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index 56a66b9..7c1edea 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -50,7 +50,7 @@ class AddColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Par
class AlterTableAddColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
identifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
val lockType: String =
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfbdf3db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
index 248f351..e14524e 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
@@ -48,7 +48,7 @@ class DropColumnPartition(rddId: Int, idx: Int, schema:
ColumnSchema) extends Pa
class AlterTableDropColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
carbonTableIdentifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
newColumns.zipWithIndex.map { column =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfbdf3db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
--
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
index 32523d8..9936a2a 100644
---
a/integration/spark-c
