HIVE-8696 : HCatClientHMSImpl doesn't use a Retrying-HiveMetastoreClient (Thiruvel Thirumoolan via Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e0044e07 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e0044e07 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e0044e07 Branch: refs/heads/beeline-cli Commit: e0044e0723d8e831ad0f29c6615b5f70e2ee0658 Parents: 26ec033 Author: Sushanth Sowmyan <khorg...@gmail.com> Authored: Thu May 7 01:12:11 2015 -0700 Committer: Sushanth Sowmyan <khorg...@gmail.com> Committed: Thu May 7 01:13:05 2015 -0700 ---------------------------------------------------------------------- .../hive/hcatalog/common/HiveClientCache.java | 9 +++- .../hcatalog/mapreduce/TestPassProperties.java | 5 +- .../hive/metastore/RetryingMetaStoreClient.java | 53 ++++++++++++++++---- 3 files changed, 55 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java index 578b6ea..0966581 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java @@ -35,6 +35,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; @@ -81,7 +82,7 @@ class HiveClientCache { } public static IMetaStoreClient getNonCachedHiveMetastoreClient(HiveConf hiveConf) throws MetaException { - return new HiveMetaStoreClient(hiveConf); + return RetryingMetaStoreClient.getProxy(hiveConf); } public HiveClientCache(HiveConf hiveConf) { @@ -226,7 +227,11 @@ class HiveClientCache { return hiveCache.get(cacheKey, new Callable<ICacheableMetaStoreClient>() { @Override public ICacheableMetaStoreClient call() throws MetaException { - return new CacheableHiveMetaStoreClient(cacheKey.getHiveConf(), timeout); + return + (ICacheableMetaStoreClient) RetryingMetaStoreClient.getProxy(cacheKey.getHiveConf(), + new Class<?>[]{HiveConf.class, Integer.class}, + new Object[]{cacheKey.getHiveConf(), timeout}, + CacheableHiveMetaStoreClient.class.getName()); } }); } catch (ExecutionException e) { http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java index 735ab5f..8673b48 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; @@ -109,8 +110,10 @@ public class TestPassProperties { new FileOutputCommitterContainer(job, null).cleanupJob(job); } catch (Exception e) { caughtException = true; - assertTrue(e.getCause().getMessage().contains( + assertTrue(((InvocationTargetException)e.getCause().getCause().getCause()).getTargetException().getMessage().contains( "Could not connect to meta store using any of the URIs provided")); + assertTrue(e.getCause().getMessage().contains( + "Unable to instantiate org.apache.hive.hcatalog.common.HiveClientCache$CacheableHiveMetaStoreClient")); } assertTrue(caughtException); } http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index 1b6487a..fb44484 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -60,6 +60,18 @@ public class RetryingMetaStoreClient implements InvocationHandler { protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { + + this(hiveConf, + new Class[] {HiveConf.class, HiveMetaHookLoader.class}, + new Object[] {hiveConf, hookLoader}, + metaCallTimeMap, + msClientClass); + } + + protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes, + Object[] constructorArgs, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) + throws MetaException { + this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES); this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); @@ -71,12 +83,14 @@ public class RetryingMetaStoreClient implements InvocationHandler { localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); - this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] { - HiveConf.class, HiveMetaHookLoader.class}, new Object[] {hiveConf, hookLoader}); + this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); } - public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) throws MetaException { - return getProxy(hiveConf, hookLoader, null, HiveMetaStoreClient.class.getName()); + public static IMetaStoreClient getProxy(HiveConf hiveConf) throws MetaException { + + return getProxy(hiveConf, new Class[]{HiveConf.class}, new Object[]{hiveConf}, null, + HiveMetaStoreClient.class.getName() + ); } public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader, @@ -84,19 +98,40 @@ public class RetryingMetaStoreClient implements InvocationHandler { return getProxy(hiveConf, hookLoader, null, mscClassName); } + public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader, + Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { + + return getProxy(hiveConf, + new Class[] {HiveConf.class, HiveMetaHookLoader.class}, + new Object[] {hiveConf, hookLoader}, + metaCallTimeMap, + mscClassName + ); + } + /** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose. */ - public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader, - Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { + public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes, + Object[] constructorArgs, String mscClassName) throws MetaException { + return getProxy(hiveConf, constructorArgTypes, constructorArgs, null, mscClassName); + } + + /** + * This constructor is meant for Hive internal use only. + * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose. + */ + public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes, + Object[] constructorArgs, Map<String, Long> metaCallTimeMap, + String mscClassName) throws MetaException { Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>) MetaStoreUtils .getClass(mscClassName); - RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, hookLoader, - metaCallTimeMap, baseClass); - + RetryingMetaStoreClient handler = + new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, + metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
