Repository: spark Updated Branches: refs/heads/branch-2.1 0e624e990 -> fa360134d
[SPARK-18507][SQL] HiveExternalCatalog.listPartitions should only call getTable once ## What changes were proposed in this pull request? HiveExternalCatalog.listPartitions should only call `getTable` once, instead of calling it for every partitions. ## How was this patch tested? N/A Author: Wenchen Fan <wenc...@databricks.com> Closes #15978 from cloud-fan/perf. (cherry picked from commit 702cd403fc8e5ce8281fe8828197ead46bdb8832) Signed-off-by: Andrew Or <andrewo...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fa360134 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fa360134 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fa360134 Branch: refs/heads/branch-2.1 Commit: fa360134d06e5bfb423f0bd769edb47dbda1d9af Parents: 0e624e9 Author: Wenchen Fan <wenc...@databricks.com> Authored: Tue Nov 22 15:25:22 2016 -0500 Committer: Andrew Or <andrewo...@gmail.com> Committed: Tue Nov 22 15:25:33 2016 -0500 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fa360134/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 5dbb402..ff0923f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -907,8 +907,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient { + val actualPartColNames = getTable(db, table).partitionColumnNames client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part => - part.copy(spec = restorePartitionSpec(part.spec, getTable(db, table).partitionColumnNames)) + part.copy(spec = restorePartitionSpec(part.spec, actualPartColNames)) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
