[CARBONDATA-2823] Support streaming property with datamap Since during query, carbondata get splits from streaming segment and columnar segments repectively, we can support streaming with index datamap.
For preaggregate datamap, it already supported streaming table, so here we will remove the outdated comments. This closes #2609 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b9e51064 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b9e51064 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b9e51064 Branch: refs/heads/external-format Commit: b9e51064017e428962d86ff8fccb3c1493e9864b Parents: abcd4f6 Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Mon Aug 6 15:34:51 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Tue Aug 7 18:21:30 2018 +0800 ---------------------------------------------------------------------- .../datamap/bloom/BloomCoarseGrainDataMapFactory.java | 2 +- .../datamap/lucene/LuceneFineGrainDataMapFactory.java | 2 +- .../apache/spark/sql/execution/strategy/DDLStrategy.scala | 8 -------- 3 files changed, 2 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java index 80a86cc..0d240c4 100644 --- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java +++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java @@ -391,7 +391,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa case ALTER_CHANGE_DATATYPE: return true; case STREAMING: - return true; + return false; case DELETE: return true; case UPDATE: http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java index 8c7539f..ff2ffdd 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java @@ -98,7 +98,7 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine case ALTER_CHANGE_DATATYPE: return true; case STREAMING: - return true; + return false; case DELETE: return true; case UPDATE: http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala index 416dbec..94b988c 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala @@ -35,7 +35,6 @@ import org.apache.spark.util.{CarbonReflectionUtils, FileUtils} import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException import org.apache.carbondata.common.logging.{LogService, LogServiceFactory} -import org.apache.carbondata.core.features.TableOperation import org.apache.carbondata.core.util.CarbonProperties /** @@ -257,8 +256,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy { if CarbonEnv.getInstance(sparkSession).carbonMetastore .tableExists(tableName)(sparkSession) => { - // TODO remove this limiation after streaming table support 'preaggregate' DataMap - // if the table has 'preaggregate' DataMap, it doesn't support streaming now val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore .lookupRelation(tableName)(sparkSession).asInstanceOf[CarbonRelation].carbonTable if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable) { @@ -266,11 +263,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy { "Unsupported operation on non transactional table") } - if (carbonTable != null && !carbonTable.canAllow(carbonTable, TableOperation.STREAMING)) { - throw new MalformedCarbonCommandException( - "streaming is not supported for index datamap") - } - // TODO remove this limitation later val property = properties.find(_._1.equalsIgnoreCase("streaming")) if (property.isDefined) {
