[HOTFIX] Changes to align printing of information in explain command based on enable.query.statistics flag
Now user need to set the flag enable.query.statistics = true to print the pruning and dataMap related information in the explain command. This closes #2795 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3a87044 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3a87044 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3a87044 Branch: refs/heads/branch-1.5 Commit: c3a8704494d8d4cffbe8f35c43101ce248913965 Parents: d8003a3 Author: manishgupta88 <tomanishgupt...@gmail.com> Authored: Wed Oct 3 18:13:45 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Wed Oct 3 20:23:12 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/profiler/ExplainCollector.java | 9 ++++++++- docs/datamap/datamap-management.md | 2 +- .../cluster/sdv/generated/BloomFilterDataMapTestCase.scala | 6 +++++- .../datamap/lucene/LuceneFineGrainDataMapSuite.scala | 5 +++++ .../lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala | 5 +++++ .../preaggregate/TestPreAggregateTableSelection.scala | 7 +++++++ .../testsuite/createTable/TestRenameTableWithDataMap.scala | 8 ++++++++ .../spark/testsuite/datamap/CGDataMapTestCase.scala | 5 +++++ .../spark/testsuite/datamap/FGDataMapTestCase.scala | 7 ++++++- .../src/main/scala/org/apache/spark/sql/CarbonSession.scala | 1 + .../sql/execution/command/table/CarbonExplainCommand.scala | 6 +++++- .../datamap/bloom/BloomCoarseGrainDataMapSuite.scala | 5 +++++ 12 files changed, 61 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java index 755c56a..8513dac 100644 --- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java @@ -26,7 +26,9 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.carbondata.common.annotations.InterfaceAudience; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo; +import org.apache.carbondata.core.util.CarbonProperties; /** * An information collector used for EXPLAIN command, to print out @@ -52,7 +54,12 @@ public class ExplainCollector { } public static void setup() { - INSTANCE = new ExplainCollector(); + boolean isQueryStatisticsEnabled = Boolean.parseBoolean(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT)); + if (isQueryStatisticsEnabled) { + INSTANCE = new ExplainCollector(); + } } public static void remove() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/docs/datamap/datamap-management.md ---------------------------------------------------------------------- diff --git a/docs/datamap/datamap-management.md b/docs/datamap/datamap-management.md index eee03a7..bf52c05 100644 --- a/docs/datamap/datamap-management.md +++ b/docs/datamap/datamap-management.md @@ -122,7 +122,7 @@ There is a DataMapCatalog interface to retrieve schema of all datamap, it can be How can user know whether datamap is used in the query? -User can use EXPLAIN command to know, it will print out something like +User can set enable.query.statistics = true and use EXPLAIN command to know, it will print out something like ```text == CarbonData Profiler == http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BloomFilterDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BloomFilterDataMapTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BloomFilterDataMapTestCase.scala index 8acbcd6..077e007 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BloomFilterDataMapTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BloomFilterDataMapTestCase.scala @@ -31,7 +31,8 @@ class BloomFilterDataMapTestCase extends QueryTest with BeforeAndAfterEach with "yyyy-MM-dd") CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss") - + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") } override protected def afterAll(): Unit = { @@ -39,6 +40,9 @@ class BloomFilterDataMapTestCase extends QueryTest with BeforeAndAfterEach with CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } private def createAllDataTypeTable(tableName: String): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index 2e3019a..a51294c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala @@ -41,6 +41,8 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { val file2 = resourcesPath + "/datamap_input.csv" override protected def beforeAll(): Unit = { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") new File(CarbonProperties.getInstance().getSystemFolderLocation).delete() LuceneFineGrainDataMapSuite.createFile(file2) sql("create database if not exists lucene") @@ -921,6 +923,9 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP, originDistributedDatamapStatus) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala index 369bed1..375495c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala @@ -42,6 +42,8 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd sqlContext.sparkSession.asInstanceOf[CarbonSession].startSearchMode() CarbonProperties .getInstance().addProperty(CarbonCommonConstants.CARBON_SEARCH_QUERY_TIMEOUT, "100s") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") LuceneFineGrainDataMapSuite.createFile(file2, n) sql("create database if not exists lucene") sql("use lucene") @@ -296,6 +298,9 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd } override protected def afterAll(): Unit = { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) LuceneFineGrainDataMapSuite.deleteFile(file2) sql("DROP TABLE IF EXISTS datamap_test") sql("DROP TABLE IF EXISTS datamap_test5") http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala index 0dc52ea..2d2bde1 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala @@ -24,7 +24,9 @@ import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row} import org.scalatest.BeforeAndAfterAll +import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES +import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.spark.util.SparkQueryTest class TestPreAggregateTableSelection extends SparkQueryTest with BeforeAndAfterAll { @@ -32,6 +34,8 @@ class TestPreAggregateTableSelection extends SparkQueryTest with BeforeAndAfterA val timeSeries = TIMESERIES.toString override def beforeAll: Unit = { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") sql("drop table if exists mainTable") sql("drop table if exists mainTableavg") sql("drop table if exists agg0") @@ -454,6 +458,9 @@ class TestPreAggregateTableSelection extends SparkQueryTest with BeforeAndAfterA sql("DROP TABLE IF EXISTS mainTableavg") sql("DROP TABLE IF EXISTS filtertable") sql("DROP TABLE IF EXISTS grouptable") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestRenameTableWithDataMap.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestRenameTableWithDataMap.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestRenameTableWithDataMap.scala index 18fb28f..78cf44b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestRenameTableWithDataMap.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestRenameTableWithDataMap.scala @@ -20,6 +20,9 @@ package org.apache.carbondata.spark.testsuite.createTable import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + /** * test functionality for alter table with datamap */ @@ -31,6 +34,8 @@ class TestRenameTableWithDataMap extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS carbon_table") sql("DROP TABLE IF EXISTS carbon_tb") sql("DROP TABLE IF EXISTS fact_table1") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") } test("Creating a bloomfilter datamap,then table rename") { @@ -188,5 +193,8 @@ class TestRenameTableWithDataMap extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS carbon_table") sql("DROP TABLE IF EXISTS carbon_tb") sql("DROP TABLE IF EXISTS fact_table1") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala index a4bc6f0..58f2542 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala @@ -361,6 +361,8 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll { //n should be about 5000000 of reset if size is default 1024 val n = 150000 CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") sql("DROP TABLE IF EXISTS normal_test") sql( """ @@ -558,6 +560,9 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS datamap_store_test") sql("DROP TABLE IF EXISTS datamap_store_test1") sql("DROP TABLE IF EXISTS datamap_store_test2") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala index 57b3672..f553a9d 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala @@ -45,7 +45,7 @@ import org.apache.carbondata.core.scan.expression.Expression import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression import org.apache.carbondata.core.scan.filter.intf.ExpressionType import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf -import org.apache.carbondata.core.util.ByteUtil +import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties} import org.apache.carbondata.core.util.path.CarbonTablePath import org.apache.carbondata.events.Event import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest @@ -437,6 +437,8 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll { //n should be about 5000000 of reset if size is default 1024 val n = 150000 CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n) + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") sql("DROP TABLE IF EXISTS normal_test") sql( """ @@ -569,5 +571,8 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS normal_test") sql("DROP TABLE IF EXISTS datamap_test") sql("DROP TABLE IF EXISTS datamap_testFG") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala index d6117de..838b28d 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala @@ -115,6 +115,7 @@ class CarbonSession(@transient val sc: SparkContext, */ @InterfaceAudience.Developer(Array("DataMap")) def isDataMapHit(sqlStatement: String, dataMapName: String): Boolean = { + // explain command will output the dataMap information only if enable.query.statistics = true val message = sql(s"EXPLAIN $sqlStatement").collect() message(0).getString(0).contains(dataMapName) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala index 0c2567a..cb402c7 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala @@ -52,7 +52,11 @@ case class CarbonExplainCommand( try { ExplainCollector.setup() queryExecution.toRdd.partitions - Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput)) + if (ExplainCollector.enabled()) { + Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput)) + } else { + Seq.empty + } } finally { ExplainCollector.remove() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3a87044/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala index 84edd73..3360530 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala @@ -41,6 +41,8 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with override protected def beforeAll(): Unit = { new File(CarbonProperties.getInstance().getSystemFolderLocation).delete() + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "true") createFile(bigFile, line = 50000) createFile(smallFile) sql(s"DROP TABLE IF EXISTS $normalTable") @@ -992,6 +994,9 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with deleteFile(smallFile) sql(s"DROP TABLE IF EXISTS $normalTable") sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, + CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT) } private def createFile(fileName: String, line: Int = 10000, start: Int = 0) = {