Repository: incubator-carbondata Updated Branches: refs/heads/master 958dfde64 -> 4cab5bf38
use recoder for all statistic log Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4e1154d5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4e1154d5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4e1154d5 Branch: refs/heads/master Commit: 4e1154d538a96d8862a2b66815cb5db1dc7e3ed5 Parents: 958dfde Author: foryou2030 <foryou2...@126.com> Authored: Sun Oct 9 16:41:20 2016 +0800 Committer: foryou2030 <foryou2...@126.com> Committed: Sun Oct 9 16:41:20 2016 +0800 ---------------------------------------------------------------------- .../core/carbon/querystatistics/QueryStatistic.java | 5 +++++ .../apache/spark/sql/optimizer/CarbonOptimizer.scala | 13 ++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4e1154d5/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java index 939b819..74fd651 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java @@ -20,6 +20,8 @@ package org.apache.carbondata.core.carbon.querystatistics; import java.io.Serializable; +import org.apache.commons.lang3.StringUtils; + /** * Wrapper class to maintain the query statistics for each phase of the query */ @@ -91,6 +93,9 @@ public class QueryStatistic implements Serializable { * @return statistic message */ public String getStatistics(String queryWithTaskId) { + if (StringUtils.isEmpty(queryWithTaskId)) { + return message + timeTaken; + } return message + " for the taskid : " + queryWithTaskId + " Is : " + timeTaken; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4e1154d5/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala index b8af85e..e428bba 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala @@ -24,16 +24,16 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.optimizer.Optimizer import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.execution.RunnableCommand import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.types.{IntegerType, StringType} import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic +import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory import org.apache.carbondata.spark.CarbonFilters /** @@ -74,10 +74,13 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation]) def apply(plan: LogicalPlan): LogicalPlan = { if (relations.nonEmpty && !isOptimized(plan)) { LOGGER.info("Starting to optimize plan") - val startTime = System.currentTimeMillis + val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder(""); + val queryStatistic = new QueryStatistic() val result = transformCarbonPlan(plan, relations) - LOGGER.statistic("Time taken for Carbon Optimizer to optimize: " + - ( System.currentTimeMillis - startTime)) + queryStatistic.addStatistics("Time taken for Carbon Optimizer to optimize: ", + System.currentTimeMillis) + recorder.recordStatistics(queryStatistic) + recorder.logStatistics() result } else { LOGGER.info("Skip CarbonOptimizer")
