Repository: hive Updated Branches: refs/heads/master b9ad6dc38 -> 92aaed243
HIVE-16014: HiveMetastoreChecker should use hive.metastore.fshandler.threads instead of hive.mv.files.thread for pool size (Vihang Karajgaonkar, reviewed by Sahil Takiar, Sergio Pena, Rajesh Balamohan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92aaed24 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92aaed24 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92aaed24 Branch: refs/heads/master Commit: 92aaed243c27c73f9168c99c2837431477c4e41c Parents: b9ad6dc Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Wed Mar 1 16:09:50 2017 -0600 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Wed Mar 1 16:09:50 2017 -0600 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/92aaed24/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java index 5d2ae2c..0859022 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java @@ -404,7 +404,12 @@ public class HiveMetaStoreChecker { */ private void checkPartitionDirs(Path basePath, Set<Path> allDirs, int maxDepth) throws IOException, HiveException { - int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 15); + // Here we just reuse the THREAD_COUNT configuration for + // METASTORE_FS_HANDLER_THREADS_COUNT since this results in better performance + // The number of missing partitions discovered are later added by metastore using a + // threadpool of size METASTORE_FS_HANDLER_THREADS_COUNT. If we have different sized + // pool here the smaller sized pool of the two becomes a bottleneck + int poolSize = conf.getInt(ConfVars.METASTORE_FS_HANDLER_THREADS_COUNT.varname, 15); // Check if too low config is provided for move files. 2x CPU is reasonable max count. poolSize = poolSize == 0 ? poolSize : Math.max(poolSize,
