Repository: incubator-carbondata Updated Branches: refs/heads/master 1d8fd0e0c -> 50ec532a2
Fixed Performance issue for dictionary loading during decoder Fixed Review Comments Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d2114d51 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d2114d51 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d2114d51 Branch: refs/heads/master Commit: d2114d51760fdf9def37920adcced54ce7856385 Parents: 1d8fd0e Author: kumarvishal <kumarvishal.1...@gmail.com> Authored: Tue Dec 27 10:52:50 2016 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Mon Mar 6 00:06:58 2017 +0800 ---------------------------------------------------------------------- .../spark/sql/CarbonDictionaryDecoder.scala | 42 +++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d2114d51/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala index 3da8299..126f0c5 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql +import scala.collection.JavaConverters._ + import org.apache.spark.TaskContext import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -215,21 +217,39 @@ case class CarbonDictionaryDecoder( } private def getDictionary(atiMap: Map[String, AbsoluteTableIdentifier], - cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = { - val dicts: Seq[Dictionary] = getDictionaryColumnIds.map { f => - if (f._2 != null) { - try { - cache.get(new DictionaryColumnUniqueIdentifier( - atiMap(f._1).getCarbonTableIdentifier, - f._2, f._3)) - } catch { - case _: Throwable => null - } + cache: Cache[DictionaryColumnUniqueIdentifier, Dictionary]) = { + val dictionaryColumnIds = getDictionaryColumnIds.map { dictionaryId => + if (dictionaryId._2 != null) { + new DictionaryColumnUniqueIdentifier( + atiMap(dictionaryId._1).getCarbonTableIdentifier, + dictionaryId._2, dictionaryId._3) } else { null } } - dicts + try { + val noDictionaryIndexes = new java.util.ArrayList[Int]() + dictionaryColumnIds.zipWithIndex.foreach { columnIndex => + if (columnIndex._1 == null) { + noDictionaryIndexes.add(columnIndex._2) + } + } + val dict = cache.getAll(dictionaryColumnIds.filter(_ != null).toSeq.asJava); + val finalDict = new java.util.ArrayList[Dictionary]() + var dictIndex: Int = 0 + dictionaryColumnIds.zipWithIndex.foreach { columnIndex => + if (!noDictionaryIndexes.contains(columnIndex._2)) { + finalDict.add(dict.get(dictIndex)) + dictIndex += 1 + } else { + finalDict.add(null) + } + } + finalDict.asScala + } catch { + case t: Throwable => Seq.empty + } + } }
