Repository: hive Updated Branches: refs/heads/master fa9a38978 -> 07492e0d2
HIVE-18347 : Allow pluggable dynamic lookup of Hive Metastores from HiveServer2 (Szehon Ho, reviewed by Vihang K) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/07492e0d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/07492e0d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/07492e0d Branch: refs/heads/master Commit: 07492e0d2f1942c1794a3190610e10207c850cf7 Parents: fa9a389 Author: Szehon Ho <szehon.apa...@gmail.com> Authored: Fri Dec 15 11:07:04 2017 +0100 Committer: Szehon Ho <szehon.apa...@gmail.com> Committed: Wed Feb 7 17:49:53 2018 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 +- .../hive/metastore/HiveMetaStoreClient.java | 110 +++++++++++++------ .../hive/metastore/conf/MetastoreConf.java | 2 + .../hive/metastore/hooks/URIResolverHook.java | 37 +++++++ 4 files changed, 118 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index eca3573..f3e74eb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -649,7 +649,8 @@ public class HiveConf extends Configuration { "Number of threads to be allocated for metastore handler for fs operations."), METASTORE_HBASE_FILE_METADATA_THREADS("hive.metastore.hbase.file.metadata.threads", 1, "Number of threads to use to read file metadata in background to cache it."), - + METASTORE_URI_RESOLVER("hive.metastore.uri.resolver", "", + "If set, fully qualified class name of resolver for hive metastore uri's"), METASTORETHRIFTCONNECTIONRETRIES("hive.metastore.connect.retries", 3, "Number of retries while opening a connection to metastore"), METASTORETHRIFTFAILURERETRIES("hive.metastore.failure.retries", 1, http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 2e76e17..23cef8d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -57,13 +57,16 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.hooks.URIResolverHook; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.thrift.TApplicationException; import org.apache.thrift.TException; @@ -112,6 +115,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { private String tokenStrForm; private final boolean localMetaStore; private final MetaStoreFilterHook filterHook; + private final URIResolverHook uriResolverHook; private final int fileMetadataBatchSize; private Map<String, String> currentMetaVars; @@ -145,6 +149,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } version = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) ? TEST_VERSION : VERSION; filterHook = loadFilterHooks(); + uriResolverHook = loadUriResolverHook(); fileMetadataBatchSize = MetastoreConf.getIntVar( conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX); @@ -172,39 +177,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { // user wants file store based configuration if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS) != null) { - String metastoreUrisString[] = MetastoreConf.getVar(conf, - ConfVars.THRIFT_URIS).split(","); - metastoreUris = new URI[metastoreUrisString.length]; - try { - int i = 0; - for (String s : metastoreUrisString) { - URI tmpUri = new URI(s); - if (tmpUri.getScheme() == null) { - throw new IllegalArgumentException("URI: " + s - + " does not have a scheme"); - } - metastoreUris[i++] = new URI( - tmpUri.getScheme(), - tmpUri.getUserInfo(), - HadoopThriftAuthBridge.getBridge().getCanonicalHostName(tmpUri.getHost()), - tmpUri.getPort(), - tmpUri.getPath(), - tmpUri.getQuery(), - tmpUri.getFragment() - ); - - } - // make metastore URIS random - if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { - List uriList = Arrays.asList(metastoreUris); - Collections.shuffle(uriList); - metastoreUris = (URI[]) uriList.toArray(); - } - } catch (IllegalArgumentException e) { - throw (e); - } catch (Exception e) { - MetaStoreUtils.logAndThrowMetaException(e); - } + resolveUris(); } else { LOG.error("NOT getting uris from conf"); throw new MetaException("MetaStoreURIs not found in conf file"); @@ -249,6 +222,51 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { open(); } + private void resolveUris() throws MetaException { + String metastoreUrisString[] = MetastoreConf.getVar(conf, + ConfVars.THRIFT_URIS).split(","); + + List<URI> metastoreURIArray = new ArrayList<URI>(); + try { + int i = 0; + for (String s : metastoreUrisString) { + URI tmpUri = new URI(s); + if (tmpUri.getScheme() == null) { + throw new IllegalArgumentException("URI: " + s + + " does not have a scheme"); + } + if (uriResolverHook != null) { + metastoreURIArray.addAll(uriResolverHook.resolveURI(tmpUri)); + } else { + metastoreURIArray.add(new URI( + tmpUri.getScheme(), + tmpUri.getUserInfo(), + HadoopThriftAuthBridge.getBridge().getCanonicalHostName(tmpUri.getHost()), + tmpUri.getPort(), + tmpUri.getPath(), + tmpUri.getQuery(), + tmpUri.getFragment() + )); + } + } + metastoreUris = new URI[metastoreURIArray.size()]; + for (int j = 0; j < metastoreURIArray.size(); j++) { + metastoreUris[j] = metastoreURIArray.get(j); + } + + if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { + List uriList = Arrays.asList(metastoreUris); + Collections.shuffle(uriList); + metastoreUris = (URI[]) uriList.toArray(); + } + } catch (IllegalArgumentException e) { + throw (e); + } catch (Exception e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + } + + private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException { Class<? extends MetaStoreFilterHook> authProviderClass = MetastoreConf. getClass(conf, ConfVars.FILTER_HOOK, DefaultMetaStoreFilterHookImpl.class, @@ -263,6 +281,26 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } } + //multiple clients may initialize the hook at the same time + synchronized private URIResolverHook loadUriResolverHook() throws IllegalStateException { + + String uriResolverClassName = + MetastoreConf.getAsString(conf, ConfVars.URI_RESOLVER); + if (uriResolverClassName.equals("")) { + return null; + } else { + LOG.info("Loading uri resolver" + uriResolverClassName); + try { + Class<?> uriResolverClass = Class.forName(uriResolverClassName, true, + JavaUtils.getClassLoader()); + return (URIResolverHook) ReflectionUtils.newInstance(uriResolverClass, null); + } catch (Exception e) { + LOG.error("Exception loading uri resolver hook" + e); + return null; + } + } + } + /** * Swaps the first element of the metastoreUris array with a random element from the * remainder of the array. @@ -324,6 +362,12 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { " at the client level."); } else { close(); + + if (uriResolverHook != null) { + //for dynamic uris, re-lookup if there are new metastore locations + resolveUris(); + } + if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { // Swap the first element of the metastoreUris[] with a random element from the rest // of the array. Rationale being that this method will generally be called when the default http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 5ef8d7a..5b8e5ca 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -777,6 +777,8 @@ public class MetastoreConf { "class is used to store and retrieve transactions and locks"), TXN_TIMEOUT("metastore.txn.timeout", "hive.txn.timeout", 300, TimeUnit.SECONDS, "time after which transactions are declared aborted if the client has not sent a heartbeat."), + URI_RESOLVER("metastore.uri.resolver", "hive.metastore.uri.resolver", "", + "If set, fully qualified class name of resolver for hive metastore uri's"), USERS_IN_ADMIN_ROLE("metastore.users.in.admin.role", "hive.users.in.admin.role", "", false, "Comma separated list of users who are in admin role for bootstrapping.\n" + "More users can be added in ADMIN role later."), http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java new file mode 100644 index 0000000..d3be5dd --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.hooks; + +import org.apache.hadoop.hive.metastore.HiveMetaException; + +import java.net.URI; +import java.util.List; + +/** + * Allows different metastore uris to be resolved. + */ +public interface URIResolverHook { + + /** + * Resolve to a proper thrift uri, or a list of uris, given uri of another scheme. + * @param uri + * @return + */ + public List<URI> resolveURI(URI uri) throws HiveMetaException; +}
