HIVE-16723 Enable configurable MetaStoreSchemaInfo (Vihang Karajgaonkar, reviewed by Sahil Takiar, Peter Vary and Naveen Gangam)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ea4807dd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ea4807dd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ea4807dd Branch: refs/heads/hive-14535 Commit: ea4807dd906462d2907a5b8feb0527423b0668c8 Parents: d85beaa Author: Naveen Gangam <[email protected]> Authored: Tue May 23 11:50:58 2017 -0400 Committer: Naveen Gangam <[email protected]> Committed: Tue May 23 11:50:58 2017 -0400 ---------------------------------------------------------------------- .../org/apache/hive/beeline/HiveSchemaTool.java | 23 ++--- .../org/apache/hadoop/hive/conf/HiveConf.java | 5 ++ .../hive/metastore/TestMetastoreVersion.java | 9 +- .../org/apache/hive/beeline/TestSchemaTool.java | 6 +- .../hive/metastore/IMetaStoreSchemaInfo.java | 92 ++++++++++++++++++++ .../hive/metastore/MetaStoreSchemaInfo.java | 39 ++++----- .../metastore/MetaStoreSchemaInfoFactory.java | 65 ++++++++++++++ .../hadoop/hive/metastore/ObjectStore.java | 7 +- .../metastore/TestMetaStoreSchemaFactory.java | 67 ++++++++++++++ .../hive/metastore/TestMetaStoreSchemaInfo.java | 19 ++-- 10 files changed, 283 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index a453580..f312e46 100644 --- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -34,7 +34,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaException; -import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.shims.ShimLoader; @@ -78,7 +79,7 @@ public class HiveSchemaTool { private final HiveConf hiveConf; private final String dbType; private final String metaDbType; - private final MetaStoreSchemaInfo metaStoreSchemaInfo; + private final IMetaStoreSchemaInfo metaStoreSchemaInfo; static final private Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName()); @@ -94,7 +95,7 @@ public class HiveSchemaTool { this.hiveConf = hiveConf; this.dbType = dbType; this.metaDbType = metaDbType; - this.metaStoreSchemaInfo = new MetaStoreSchemaInfo(hiveHome, dbType); + this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, hiveHome, dbType); } public HiveConf getHiveConf() { @@ -167,7 +168,7 @@ public class HiveSchemaTool { */ public void showInfo() throws HiveMetaException { Connection metastoreConn = getConnectionToMetastore(true); - String hiveVersion = MetaStoreSchemaInfo.getHiveSchemaVersion(); + String hiveVersion = metaStoreSchemaInfo.getHiveSchemaVersion(); String dbVersion = getMetaStoreSchemaVersion(metastoreConn); System.out.println("Hive distribution version:\t " + hiveVersion); System.out.println("Metastore schema version:\t " + dbVersion); @@ -499,13 +500,13 @@ public class HiveSchemaTool { String newSchemaVersion = getMetaStoreSchemaVersion( getConnectionToMetastore(false)); // verify that the new version is added to schema - assertCompatibleVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); + assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); } private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion) throws HiveMetaException { - if (!MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) { + if (!metaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) { throw new HiveMetaException("Metastore schema version is not compatible. Hive Version: " + hiveSchemaVersion + ", Database Schema Version: " + dbSchemaVersion); } @@ -533,7 +534,7 @@ public class HiveSchemaTool { * @throws MetaException */ public void doUpgrade(String fromSchemaVer) throws HiveMetaException { - if (MetaStoreSchemaInfo.getHiveSchemaVersion().equals(fromSchemaVer)) { + if (metaStoreSchemaInfo.getHiveSchemaVersion().equals(fromSchemaVer)) { System.out.println("No schema upgrade required from version " + fromSchemaVer); return; } @@ -542,7 +543,7 @@ public class HiveSchemaTool { metaStoreSchemaInfo.getUpgradeScripts(fromSchemaVer); testConnectionToMetastore(); System.out.println("Starting upgrade metastore schema from version " + - fromSchemaVer + " to " + MetaStoreSchemaInfo.getHiveSchemaVersion()); + fromSchemaVer + " to " + metaStoreSchemaInfo.getHiveSchemaVersion()); String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir(); try { for (String scriptFile : upgradeScripts) { @@ -568,7 +569,7 @@ public class HiveSchemaTool { * @throws MetaException */ public void doInit() throws HiveMetaException { - doInit(MetaStoreSchemaInfo.getHiveSchemaVersion()); + doInit(metaStoreSchemaInfo.getHiveSchemaVersion()); // Revalidated the new version after upgrade verifySchemaVersion(); @@ -715,7 +716,7 @@ public class HiveSchemaTool { System.out.println("Validating schema version"); try { String newSchemaVersion = getMetaStoreSchemaVersion(conn, true); - assertCompatibleVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); + assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); } catch (HiveMetaException hme) { if (hme.getMessage().contains("Metastore schema version is not compatible") || hme.getMessage().contains("Multiple versions were found in metastore") @@ -927,7 +928,7 @@ public class HiveSchemaTool { private void runPreUpgrade(String scriptDir, String scriptFile) { for (int i = 0;; i++) { String preUpgradeScript = - MetaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile); + metaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile); File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); if (!preUpgradeScriptFile.isFile()) { break; http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7dedd23..06332ac 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -777,6 +777,11 @@ public class HiveConf extends Configuration { METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", false, "When true the current MS version is recorded in the VERSION table. If this is disabled and verification is\n" + " enabled the MS will be unusable."), + METASTORE_SCHEMA_INFO_CLASS("hive.metastore.schema.info.class", + "org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo", + "Fully qualified class name for the metastore schema information class \n" + + "which is used by schematool to fetch the schema information.\n" + + " This class should implement the IMetaStoreSchemaInfo interface"), METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed", "Default transaction isolation level for identity generation."), METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false, http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 7188af6..0a034d3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -41,6 +41,7 @@ public class TestMetastoreVersion extends TestCase { private Driver driver; private String metaStoreRoot; private String testMetastoreDB; + private IMetaStoreSchemaInfo metastoreSchemaInfo; @Override protected void setUp() throws Exception { @@ -63,6 +64,8 @@ public class TestMetastoreVersion extends TestCase { System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + testMetastoreDB + ";create=true"); metaStoreRoot = System.getProperty("test.tmp.dir"); + metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, + System.getProperty("test.tmp.dir", "target/tmp"), "derby"); } @Override @@ -121,7 +124,7 @@ public class TestMetastoreVersion extends TestCase { driver.run("show tables"); // correct version stored by Metastore during startup - assertEquals(MetaStoreSchemaInfo.getHiveSchemaVersion(), getVersion(hiveConf)); + assertEquals(metastoreSchemaInfo.getHiveSchemaVersion(), getVersion(hiveConf)); setVersion(hiveConf, "foo"); assertEquals("foo", getVersion(hiveConf)); } @@ -139,7 +142,7 @@ public class TestMetastoreVersion extends TestCase { ObjectStore.setSchemaVerified(false); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, true); - setVersion(hiveConf, MetaStoreSchemaInfo.getHiveSchemaVersion()); + setVersion(hiveConf, metastoreSchemaInfo.getHiveSchemaVersion()); driver = new Driver(hiveConf); CommandProcessorResponse proc = driver.run("show tables"); assertTrue(proc.getResponseCode() == 0); @@ -189,13 +192,11 @@ public class TestMetastoreVersion extends TestCase { // write the given version to metastore private String getVersion(HiveConf conf) throws HiveMetaException { - MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby"); return getMetaStoreVersion(); } // write the given version to metastore private void setVersion(HiveConf conf, String version) throws HiveMetaException { - MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(metaStoreRoot, "derby"); setMetaStoreVersion(version, "setVersion test"); } http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 32f5903..d1658f7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -35,7 +35,9 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser; @@ -219,7 +221,9 @@ public class TestSchemaTool extends TestCase { * @throws Exception */ public void testSchemaInit() throws Exception { - schemaTool.doInit(MetaStoreSchemaInfo.getHiveSchemaVersion()); + IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, + System.getProperty("test.tmp.dir", "target/tmp"), "derby"); + schemaTool.doInit(metastoreSchemaInfo.getHiveSchemaVersion()); schemaTool.verifySchemaVersion(); } http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java new file mode 100644 index 0000000..d662743 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import java.util.List; + +/** + * Defines the method which must be implemented to be used using schema tool to support metastore + * schema upgrades. The configuration hive.metastore.schema.info.class is used to create instances + * of this type by SchemaTool. + * + * Instances of this interface should be created using MetaStoreSchemaInfoFactory class which uses + * two Strings argument constructor to instantiate the implementations of this interface + */ [email protected] +public interface IMetaStoreSchemaInfo { + String SQL_FILE_EXTENSION = ".sql"; + + /*** + * Get the list of sql scripts required to upgrade from the give version to current. + * + * @param fromVersion + * @return + * @throws HiveMetaException + */ + List<String> getUpgradeScripts(String fromVersion) throws HiveMetaException; + + /*** + * Get the name of the script to initialize the schema for given version + * + * @param toVersion Target version. If it's null, then the current server version is used + * @return + * @throws HiveMetaException + */ + String generateInitFileName(String toVersion) throws HiveMetaException; + + /** + * Find the directory of metastore scripts + * + * @return the path of directory where the sql scripts are + */ + String getMetaStoreScriptDir(); + + /** + * Get the pre-upgrade script for a given script name. Schema tool runs the pre-upgrade scripts + * returned by this method before running any upgrade scripts. These scripts could contain setup + * statements may fail on some database versions and failure is ignorable. + * + * @param index - index number of the file. The preupgrade script name is derived using the given + * index + * @param scriptName - upgrade script name + * @return name of the pre-upgrade script to be run before running upgrade script + */ + String getPreUpgradeScriptName(int index, String scriptName); + + /** + * Get hive distribution schema version. Schematool uses this version to identify + * the Hive version. It compares this version with the version found in metastore database + * to determine the upgrade or initialization scripts + * @return Hive schema version + */ + String getHiveSchemaVersion(); + + /** + * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This + * is result of the db schema upgrade design principles followed in hive project. The state where + * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or + * 'rolling downgrade' is happening. This is a state where hive is functional and returning non + * zero status for it is misleading. + * + * @param hiveVersion version of hive software + * @param dbVersion version of metastore rdbms schema + * @return true if versions are compatible + */ + boolean isVersionCompatible(String productVersion, String dbVersion); +} http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index 320902b..6bddb8e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -31,14 +31,13 @@ import org.apache.hive.common.util.HiveVersionInfo; import com.google.common.collect.ImmutableMap; -public class MetaStoreSchemaInfo { - private static final String SQL_FILE_EXTENSION = ".sql"; - private static final String UPGRADE_FILE_PREFIX = "upgrade-"; +public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { + protected static final String UPGRADE_FILE_PREFIX = "upgrade-"; private static final String INIT_FILE_PREFIX = "hive-schema-"; private static final String VERSION_UPGRADE_LIST = "upgrade.order"; private static final String PRE_UPGRADE_PREFIX = "pre-"; - private final String dbType; - private final String hiveSchemaVersions[]; + protected final String dbType; + private String[] hiveSchemaVersions; private final String hiveHome; // Some version upgrades often don't change schema. So they are equivalent to @@ -55,6 +54,9 @@ public class MetaStoreSchemaInfo { public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaException { this.hiveHome = hiveHome; this.dbType = dbType; + } + + private void loadAllUpgradeScripts(String dbType) throws HiveMetaException { // load upgrade order for the given dbType List<String> upgradeOrderList = new ArrayList<String>(); String upgradeListFile = getMetaStoreScriptDir() + File.separator + @@ -79,6 +81,7 @@ public class MetaStoreSchemaInfo { * @return * @throws HiveMetaException */ + @Override public List<String> getUpgradeScripts(String fromVersion) throws HiveMetaException { List <String> upgradeScriptList = new ArrayList<String>(); @@ -87,6 +90,7 @@ public class MetaStoreSchemaInfo { if (getHiveSchemaVersion().equals(fromVersion)) { return upgradeScriptList; } + loadAllUpgradeScripts(dbType); // Find the list of scripts to execute for this upgrade int firstScript = hiveSchemaVersions.length; for (int i=0; i < hiveSchemaVersions.length; i++) { @@ -112,6 +116,7 @@ public class MetaStoreSchemaInfo { * @return * @throws HiveMetaException */ + @Override public String generateInitFileName(String toVersion) throws HiveMetaException { if (toVersion == null) { toVersion = getHiveSchemaVersion(); @@ -130,6 +135,7 @@ public class MetaStoreSchemaInfo { * Find the directory of metastore scripts * @return */ + @Override public String getMetaStoreScriptDir() { return hiveHome + File.separatorChar + "scripts" + File.separatorChar + "metastore" + @@ -141,11 +147,13 @@ public class MetaStoreSchemaInfo { return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION; } - public static String getPreUpgradeScriptName(int index, String upgradeScriptName) { + @Override + public String getPreUpgradeScriptName(int index, String upgradeScriptName) { return PRE_UPGRADE_PREFIX + index + "-" + upgradeScriptName; } - public static String getHiveSchemaVersion() { + @Override + public String getHiveSchemaVersion() { String hiveVersion = HiveVersionInfo.getShortVersion(); return getEquivalentVersion(hiveVersion); } @@ -160,21 +168,8 @@ public class MetaStoreSchemaInfo { } } - /** - * A dbVersion is compatible with hive version if it is greater or equal to - * the hive version. This is result of the db schema upgrade design principles - * followed in hive project. The state where db schema version is ahead of - * hive software version is often seen when a 'rolling upgrade' or - * 'rolling downgrade' is happening. This is a state where hive is functional - * and returning non zero status for it is misleading. - * - * @param hiveVersion - * version of hive software - * @param dbVersion - * version of metastore rdbms schema - * @return true if versions are compatible - */ - public static boolean isVersionCompatible(String hiveVersion, String dbVersion) { + @Override + public boolean isVersionCompatible(String hiveVersion, String dbVersion) { hiveVersion = getEquivalentVersion(hiveVersion); dbVersion = getEquivalentVersion(dbVersion); if (hiveVersion.equals(dbVersion)) { http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java new file mode 100644 index 0000000..1133cf2 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Factory class implementation to create instances of IMetaStoreSchemaInfo + * based on the provided configuration + */ +public class MetaStoreSchemaInfoFactory { + public static final Logger LOG = LoggerFactory.getLogger(MetaStoreSchemaInfoFactory.class); + + public static IMetaStoreSchemaInfo get(Configuration conf) { + String hiveHome = System.getenv("HIVE_HOME"); + if (hiveHome == null) { + LOG.debug("HIVE_HOME is not set. Using current directory instead"); + hiveHome = "."; + } + return get(conf, hiveHome, null); + } + + public static IMetaStoreSchemaInfo get(Configuration conf, String hiveHome, String dbType) { + String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, + HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.defaultStrVal); + Class<?> clasz = null; + try { + clasz = conf.getClassByName(className); + } catch (ClassNotFoundException e) { + LOG.error("Unable to load class " + className, e); + throw new IllegalArgumentException(e); + } + Constructor<?> constructor = null; + try { + constructor = clasz.getConstructor(String.class, String.class); + constructor.setAccessible(true); + return (IMetaStoreSchemaInfo) constructor.newInstance(hiveHome, dbType); + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + LOG.error("Unable to create instance of class " + className, e); + throw new IllegalArgumentException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 19becb8..28b1e57 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -7666,7 +7666,8 @@ public class ObjectStore implements RawStore, Configurable { // read the schema version stored in metastore db String dbSchemaVer = getMetaStoreSchemaVersion(); // version of schema for this version of hive - String hiveSchemaVer = MetaStoreSchemaInfo.getHiveSchemaVersion(); + IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(getConf()); + String hiveSchemaVer = metastoreSchemaInfo.getHiveSchemaVersion(); if (dbSchemaVer == null) { if (strictValidation) { @@ -7680,7 +7681,7 @@ public class ObjectStore implements RawStore, Configurable { "Set by MetaStore " + USER + "@" + HOSTNAME); } } else { - if (MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) { + if (metastoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) { LOG.debug("Found expected HMS version of " + dbSchemaVer); } else { // metastore schema version is different than Hive distribution needs @@ -7729,7 +7730,7 @@ public class ObjectStore implements RawStore, Configurable { } catch (JDODataStoreException e) { if (e.getCause() instanceof MissingTableException) { throw new MetaException("Version table not found. " + "The metastore is not upgraded to " - + MetaStoreSchemaInfo.getHiveSchemaVersion()); + + MetaStoreSchemaInfoFactory.get(getConf()).getHiveSchemaVersion()); } else { throw MetaStoreUtils.newMetaException(e); } http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java new file mode 100644 index 0000000..99eec40 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestMetaStoreSchemaFactory { + private HiveConf conf; + + @Before + public void setup() { + conf = new HiveConf(this.getClass()); + } + + @Test + public void testDefaultConfig() { + IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf); + Assert.assertNotNull(metastoreSchemaInfo); + } + + @Test + public void testWithConfigSet() { + conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, + MetaStoreSchemaInfo.class.getCanonicalName()); + IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf); + Assert.assertNotNull(metastoreSchemaInfo); + Assert.assertTrue("Unexpected instance type of the class MetaStoreSchemaInfo", + metastoreSchemaInfo instanceof MetaStoreSchemaInfo); + } + + @Test + public void testConstructor() { + String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, + MetaStoreSchemaInfo.class.getCanonicalName()); + Class<?> clasz = null; + try { + clasz = conf.getClassByName(className); + clasz.getConstructor(String.class, String.class); + } catch (NoSuchMethodException | IllegalArgumentException | ClassNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidClassName() { + conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, "invalid.class.name"); + MetaStoreSchemaInfoFactory.get(conf); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ea4807dd/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java index 7142001..35e862f 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.conf.HiveConf; import org.junit.Assert; import org.junit.Test; @@ -31,18 +32,20 @@ public class TestMetaStoreSchemaInfo { // first argument is hiveVersion, it is compatible if 2nd argument - dbVersion is // greater than or equal to it // check the compatible case - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1")); - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2")); - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1")); - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0")); + IMetaStoreSchemaInfo metastoreSchemaInfo = + MetaStoreSchemaInfoFactory.get(new HiveConf(TestMetaStoreSchemaInfo.class)); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1")); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2")); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1")); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0")); // check equivalent versions, should be compatible - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1")); - Assert.assertTrue(MetaStoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0")); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1")); + Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0")); // check incompatible versions - Assert.assertFalse(MetaStoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0")); - Assert.assertFalse(MetaStoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0")); + Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0")); + Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0")); }
