Repository: incubator-carbondata Updated Branches: refs/heads/master 76c059a29 -> 1474bbf9a
Save useInvertedIndex info into thrift store Save useInvertedIndex info into thrift store Fix the judge of no_dic_col add commont style Add comment Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/96a75b30 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/96a75b30 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/96a75b30 Branch: refs/heads/master Commit: 96a75b30ae327475d22db152eaed7853dca9799c Parents: 76c059a Author: Zhangshunyu <zhangshu...@huawei.com> Authored: Thu Sep 8 15:48:03 2016 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Mon Feb 27 23:48:29 2017 +0800 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 2 ++ .../metadata/schema/table/column/ColumnSchema.java | 17 ++++++++++------- .../carbondata/core/util/CarbonMetadataUtil.java | 4 +++- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 ++-- .../csvbased/CarbonCSVBasedSeqGenStep.java | 6 +++++- 5 files changed, 22 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 146b78e..9e890ef 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -804,6 +804,8 @@ public final class CarbonCommonConstants { public static final String COLUMN_PROPERTIES = "columnproperties"; // table block size in MB public static final String TABLE_BLOCKSIZE = "table_blocksize"; + // set in column level to disable inverted index + public static final String NO_INVERTED_INDEX = "no_inverted_index"; /** * this variable is to enable/disable identify high cardinality during first data loading http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java index 54114d3..7aa0900 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java @@ -73,11 +73,6 @@ public class ColumnSchema implements Serializable { private boolean isDimensionColumn; /** - * Whether the column should use inverted index - */ - private boolean useInvertedIndex = true; - - /** * The group ID for column used for row format columns, * where in columns in each group are chunked together. */ @@ -174,14 +169,22 @@ public class ColumnSchema implements Serializable { * the isUseInvertedIndex */ public boolean isUseInvertedIndex() { - return useInvertedIndex; + return this.hasEncoding(Encoding.INVERTED_INDEX); } /** * @param useInvertedIndex the useInvertedIndex to set */ public void setUseInvertedIndex(boolean useInvertedIndex) { - this.useInvertedIndex = useInvertedIndex; + if (useInvertedIndex) { + if (!hasEncoding(Encoding.INVERTED_INDEX)) { + this.getEncodingList().add(Encoding.INVERTED_INDEX); + } + } else { + if (hasEncoding(Encoding.INVERTED_INDEX)) { + this.getEncodingList().remove(Encoding.INVERTED_INDEX); + } + } } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 55c0302..371855b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -345,7 +345,9 @@ public class CarbonMetadataUtil { if (!isSortedKeyColumn[i]) { dataChunk.setRowid_page_offset(blockletInfoColumnar.getKeyBlockIndexOffSets()[j]); dataChunk.setRowid_page_length(blockletInfoColumnar.getKeyBlockIndexLength()[j]); - encodings.add(Encoding.INVERTED_INDEX); + if (!encodings.contains(Encoding.INVERTED_INDEX)) { + encodings.add(Encoding.INVERTED_INDEX); + } j++; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index 742a145..9bfa8a9 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -448,9 +448,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { var noInvertedIdxColsProps: Array[String] = Array[String]() var noInvertedIdxCols: Seq[String] = Seq[String]() - if (tableProperties.get("NO_INVERTED_INDEX").isDefined) { + if (tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).isDefined) { noInvertedIdxColsProps = - tableProperties.get("NO_INVERTED_INDEX").get.split(',').map(_.trim) + tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).get.split(',').map(_.trim) noInvertedIdxColsProps.foreach { noInvertedIdxColProp => if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) { val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp + http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96a75b30/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java index 6e0a4a4..3edd43c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java @@ -51,6 +51,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory; import org.apache.carbondata.core.metadata.CarbonMetadata; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; @@ -1933,7 +1934,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { dimListExcludingNoDictionaryColumn = new ArrayList<>(dimensionsList.size() - meta.noDictionaryCols.length); for (CarbonDimension dimension : dimensionsList) { - if (!dimension.getEncoder().isEmpty()) { + // Here if dimension.getEncoder() lnly contains Encoding.INVERTED_INDEX, it + // means that NoDicColumn using InvertedIndex, so not put it into dic dims list. + if (!dimension.getEncoder().isEmpty() && !((1 == dimension.getEncoder().size()) && + dimension.getEncoder().contains(Encoding.INVERTED_INDEX))) { dimListExcludingNoDictionaryColumn.add(dimension); } }