Repository: spark Updated Branches: refs/heads/master 77d046ec4 -> b3a07526f
[SPARK-21718][SQL] Heavy log of type: "Skipping partition based on stats ..." ## What changes were proposed in this pull request? Reduce 'Skipping partitions' message to debug ## How was this patch tested? Existing tests Author: Sean Owen <so...@cloudera.com> Closes #19010 from srowen/SPARK-21718. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b3a07526 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b3a07526 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b3a07526 Branch: refs/heads/master Commit: b3a07526fe774fd64fe3a2b9a2381eff9a3c49a3 Parents: 77d046e Author: Sean Owen <so...@cloudera.com> Authored: Mon Aug 21 14:20:40 2017 +0200 Committer: Herman van Hovell <hvanhov...@databricks.com> Committed: Mon Aug 21 14:20:40 2017 +0200 ---------------------------------------------------------------------- .../sql/execution/columnar/InMemoryTableScanExec.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/b3a07526/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala index 1d60137..c7ddec5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala @@ -166,12 +166,13 @@ case class InMemoryTableScanExec( if (inMemoryPartitionPruningEnabled) { cachedBatchIterator.filter { cachedBatch => if (!partitionFilter.eval(cachedBatch.stats)) { - def statsString: String = schemaIndex.map { - case (a, i) => + logDebug { + val statsString = schemaIndex.map { case (a, i) => val value = cachedBatch.stats.get(i, a.dataType) s"${a.name}: $value" - }.mkString(", ") - logInfo(s"Skipping partition based on stats $statsString") + }.mkString(", ") + s"Skipping partition based on stats $statsString" + } false } else { true --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org