Repository: hive Updated Branches: refs/heads/branch-3 e20592cb2 -> 89382aee7
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java new file mode 100644 index 0000000..cd1d57b --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; + +/** + * Print Hive version and schema version. + */ +class SchemaToolTaskInfo extends SchemaToolTask { + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + // do nothing + } + + @Override + void execute() throws HiveMetaException { + String hiveVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion(); + MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true); + String dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); + + System.out.println("Hive distribution version:\t " + hiveVersion); + System.out.println("Metastore schema version:\t " + dbVersion); + + schemaTool.assertCompatibleVersion(hiveVersion, dbVersion); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java new file mode 100644 index 0000000..e3fa495 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import java.io.IOException; + +import org.apache.hadoop.hive.metastore.HiveMetaException; + +/** + * Initialize the metastore schema. + */ +class SchemaToolTaskInit extends SchemaToolTask { + private boolean validate = true; + private String toVersion; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + if (cl.hasOption("initSchemaTo")) { + this.toVersion = cl.getOptionValue("initSchemaTo"); + this.validate = false; + } + } + + private void ensureToVersion() throws HiveMetaException { + if (toVersion != null) { + return; + } + + // If null then current hive version is used + toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion(); + System.out.println("Initializing the schema to: " + toVersion); + } + + @Override + void execute() throws HiveMetaException { + ensureToVersion(); + + schemaTool.testConnectionToMetastore(); + System.out.println("Starting metastore schema initialization to " + toVersion); + + String initScriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); + String initScriptFile = schemaTool.getMetaStoreSchemaInfo().generateInitFileName(toVersion); + + try { + System.out.println("Initialization script " + initScriptFile); + if (!schemaTool.isDryRun()) { + schemaTool.execSql(initScriptDir, initScriptFile); + System.out.println("Initialization script completed"); + } + } catch (IOException e) { + throw new HiveMetaException("Schema initialization FAILED! Metastore state would be inconsistent!", e); + } + + if (validate) { + schemaTool.verifySchemaVersion(); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java new file mode 100644 index 0000000..8a9b9d1 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Print Hive version and schema version. + */ +class SchemaToolTaskMoveDatabase extends SchemaToolTask { + private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveDatabase.class.getName()); + + private String fromCatName; + private String toCatName; + private String dbName; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + fromCatName = normalizeIdentifier(cl.getOptionValue("fromCatalog")); + toCatName = normalizeIdentifier(cl.getOptionValue("toCatalog")); + dbName = normalizeIdentifier(cl.getOptionValue("moveDatabase")); + } + + @Override + void execute() throws HiveMetaException { + System.out.println(String.format("Moving database %s from catalog %s to catalog %s", + dbName, fromCatName, toCatName)); + Connection conn = schemaTool.getConnectionToMetastore(true); + boolean success = false; + try { + conn.setAutoCommit(false); + try (Statement stmt = conn.createStatement()) { + updateCatalogNameInTable(stmt, "DBS", "CTLG_NAME", "NAME", fromCatName, toCatName, dbName, false); + updateCatalogNameInTable(stmt, "TAB_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true); + updateCatalogNameInTable(stmt, "PART_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true); + updateCatalogNameInTable(stmt, "PARTITION_EVENTS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true); + updateCatalogNameInTable(stmt, "NOTIFICATION_LOG", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true); + conn.commit(); + success = true; + } + } catch (SQLException e) { + throw new HiveMetaException("Failed to move database", e); + } finally { + try { + if (!success) { + conn.rollback(); + } + } catch (SQLException e) { + // Not really much we can do here. + LOG.error("Failed to rollback, everything will probably go bad from here."); + } + } + } + + private static final String UPDATE_CATALOG_NAME_STMT = + "update <q>%s<q> " + + " set <q>%s<q> = '%s' " + + " where <q>%s<q> = '%s' " + + " and <q>%s<q> = '%s'"; + + private void updateCatalogNameInTable(Statement stmt, String tableName, String catColName, String dbColName, + String fromCatName, String toCatName, String dbName, boolean zeroUpdatesOk) + throws HiveMetaException, SQLException { + String update = String.format(schemaTool.quote(UPDATE_CATALOG_NAME_STMT), tableName, catColName, toCatName, + catColName, fromCatName, dbColName, dbName); + LOG.debug("Going to run " + update); + int numUpdated = stmt.executeUpdate(update); + if (numUpdated != 1 && !(zeroUpdatesOk && numUpdated == 0)) { + throw new HiveMetaException("Failed to properly update the " + tableName + + " table. Expected to update 1 row but instead updated " + numUpdated); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java new file mode 100644 index 0000000..a8f9228 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Print Hive version and schema version. + */ +class SchemaToolTaskMoveTable extends SchemaToolTask { + private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveTable.class.getName()); + + private String fromCat; + private String toCat; + private String fromDb; + private String toDb; + private String tableName; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + fromCat = normalizeIdentifier(cl.getOptionValue("fromCatalog")); + toCat = normalizeIdentifier(cl.getOptionValue("toCatalog")); + fromDb = normalizeIdentifier(cl.getOptionValue("fromDatabase")); + toDb = normalizeIdentifier(cl.getOptionValue("toDatabase")); + tableName = normalizeIdentifier(cl.getOptionValue("moveTable")); + } + + @Override + void execute() throws HiveMetaException { + Connection conn = schemaTool.getConnectionToMetastore(true); + boolean success = false; + try { + conn.setAutoCommit(false); + try (Statement stmt = conn.createStatement()) { + updateTableId(stmt); + updateDbNameForTable(stmt, "TAB_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName); + updateDbNameForTable(stmt, "PART_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName); + updateDbNameForTable(stmt, "PARTITION_EVENTS", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName); + updateDbNameForTable(stmt, "NOTIFICATION_LOG", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName); + conn.commit(); + success = true; + } + } catch (SQLException se) { + throw new HiveMetaException("Failed to move table", se); + } finally { + try { + if (!success) { + conn.rollback(); + } + } catch (SQLException e) { + // Not really much we can do here. + LOG.error("Failed to rollback, everything will probably go bad from here."); + } + + } + } + + private static final String UPDATE_TABLE_ID_STMT = + "update <q>TBLS<q> " + + " set <q>DB_ID<q> = %d " + + " where <q>DB_ID<q> = %d " + + " and <q>TBL_NAME<q> = '%s'"; + + private void updateTableId(Statement stmt) throws SQLException, HiveMetaException { + // Find the old database id + long oldDbId = getDbId(stmt, fromDb, fromCat); + + // Find the new database id + long newDbId = getDbId(stmt, toDb, toCat); + + String update = String.format(schemaTool.quote(UPDATE_TABLE_ID_STMT), newDbId, oldDbId, tableName); + LOG.debug("Going to run " + update); + int numUpdated = stmt.executeUpdate(update); + if (numUpdated != 1) { + throw new HiveMetaException( + "Failed to properly update TBLS table. Expected to update " + + "1 row but instead updated " + numUpdated); + } + } + + private static final String DB_ID_QUERY = + "select <q>DB_ID<q> " + + " from <q>DBS<q> " + + " where <q>NAME<q> = '%s' " + + " and <q>CTLG_NAME<q> = '%s'"; + + private long getDbId(Statement stmt, String db, String catalog) throws SQLException, HiveMetaException { + String query = String.format(schemaTool.quote(DB_ID_QUERY), db, catalog); + LOG.debug("Going to run " + query); + try (ResultSet rs = stmt.executeQuery(query)) { + if (!rs.next()) { + throw new HiveMetaException("Unable to find database " + fromDb); + } + return rs.getLong(1); + } + } + + private static final String UPDATE_DB_NAME_STMT = + "update <q>%s<q> " + + " set <q>CAT_NAME<q> = '%s', " + + " <q>DB_NAME<q> = '%s' " + + " where <q>CAT_NAME<q> = '%s' " + + " and <q>DB_NAME<q> = '%s' " + + " and <q>%s<q> = '%s'"; + + private void updateDbNameForTable(Statement stmt, String tableName, String tableColumnName, String fromCat, + String toCat, String fromDb, String toDb, String hiveTblName) throws HiveMetaException, SQLException { + String update = String.format(schemaTool.quote(UPDATE_DB_NAME_STMT), tableName, toCat, toDb, fromCat, fromDb, + tableColumnName, hiveTblName); + + LOG.debug("Going to run " + update); + int numUpdated = stmt.executeUpdate(update); + if (numUpdated > 1 || numUpdated < 0) { + throw new HiveMetaException("Failed to properly update the " + tableName + + " table. Expected to update 1 row but instead updated " + numUpdated); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java new file mode 100644 index 0000000..5e71609 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; + +/** + * Perform metastore schema upgrade. + */ +class SchemaToolTaskUpgrade extends SchemaToolTask { + private String fromVersion; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + if (cl.hasOption("upgradeSchemaFrom")) { + this.fromVersion = cl.getOptionValue("upgradeSchemaFrom"); + } + } + + private void ensureFromVersion() throws HiveMetaException { + if (fromVersion != null) { + return; + } + + // If null, then read from the metastore + MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false); + fromVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); + if (fromVersion == null || fromVersion.isEmpty()) { + throw new HiveMetaException("Schema version not stored in the metastore. " + + "Metastore schema is too old or corrupt. Try specifying the version manually"); + } + System.out.println("Upgrading from the version " + fromVersion); + } + + @Override + void execute() throws HiveMetaException { + ensureFromVersion(); + + if (schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion().equals(fromVersion)) { + System.out.println("No schema upgrade required from version " + fromVersion); + return; + } + + // Find the list of scripts to execute for this upgrade + List<String> upgradeScripts = schemaTool.getMetaStoreSchemaInfo().getUpgradeScripts(fromVersion); + schemaTool.testConnectionToMetastore(); + System.out.println("Starting upgrade metastore schema from version " + fromVersion + " to " + + schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion()); + String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); + try { + for (String scriptFile : upgradeScripts) { + System.out.println("Upgrade script " + scriptFile); + if (!schemaTool.isDryRun()) { + runPreUpgrade(scriptDir, scriptFile); + schemaTool.execSql(scriptDir, scriptFile); + System.out.println("Completed " + scriptFile); + } + } + } catch (IOException e) { + throw new HiveMetaException("Upgrade FAILED! Metastore state would be inconsistent !!", e); + } + + // Revalidated the new version after upgrade + schemaTool.verifySchemaVersion(); + } + + /** + * Run pre-upgrade scripts corresponding to a given upgrade script, + * if any exist. The errors from pre-upgrade are ignored. + * Pre-upgrade scripts typically contain setup statements which + * may fail on some database versions and failure is ignorable. + * + * @param scriptDir upgrade script directory name + * @param scriptFile upgrade script file name + */ + private void runPreUpgrade(String scriptDir, String scriptFile) { + for (int i = 0;; i++) { + String preUpgradeScript = schemaTool.getMetaStoreSchemaInfo().getPreUpgradeScriptName(i, scriptFile); + File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); + if (!preUpgradeScriptFile.isFile()) { + break; + } + + try { + schemaTool.execSql(scriptDir, preUpgradeScript); + System.out.println("Completed " + preUpgradeScript); + } catch (Exception e) { + // Ignore the pre-upgrade script errors + System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage()); + if (schemaTool.isVerbose()) { + e.printStackTrace(); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java new file mode 100644 index 0000000..d86b457 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; + +/** + * Print Hive version and schema version. + */ +class SchemaToolTaskValidate extends SchemaToolTask { + private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskValidate.class.getName()); + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + // do nothing + } + + @Override + void execute() throws HiveMetaException { + System.out.println("Starting metastore validation\n"); + Connection conn = schemaTool.getConnectionToMetastore(false); + boolean success = true; + try { + success &= validateSchemaVersions(); + success &= validateSequences(conn); + success &= validateSchemaTables(conn); + success &= validateLocations(conn, schemaTool.getValidationServers()); + success &= validateColumnNullValues(conn); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + throw new HiveMetaException("Failed to close metastore connection", e); + } + } + } + + System.out.print("Done with metastore validation: "); + if (!success) { + System.out.println("[FAIL]"); + throw new HiveMetaException("Validation failed"); + } else { + System.out.println("[SUCCESS]"); + } + } + + boolean validateSchemaVersions() throws HiveMetaException { + System.out.println("Validating schema version"); + try { + String hiveSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion(); + MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false); + String newSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); + schemaTool.assertCompatibleVersion(hiveSchemaVersion, newSchemaVersion); + } catch (HiveMetaException hme) { + if (hme.getMessage().contains("Metastore schema version is not compatible") || + hme.getMessage().contains("Multiple versions were found in metastore") || + hme.getMessage().contains("Could not find version info in metastore VERSION table")) { + System.err.println(hme.getMessage()); + System.out.println("[FAIL]\n"); + return false; + } else { + throw hme; + } + } + System.out.println("[SUCCESS]\n"); + return true; + } + + private static final String QUERY_SEQ = + " select t.<q>NEXT_VAL<q>" + + " from <q>SEQUENCE_TABLE<q> t " + + " where t.<q>SEQUENCE_NAME<q> = ? " + + "order by t.<q>SEQUENCE_NAME<q>"; + + private static final String QUERY_MAX_ID = + "select max(<q>%s<q>)" + + " from <q>%s<q>"; + + @VisibleForTesting + boolean validateSequences(Connection conn) throws HiveMetaException { + Map<String, Pair<String, String>> seqNameToTable = + new ImmutableMap.Builder<String, Pair<String, String>>() + .put("MDatabase", Pair.of("DBS", "DB_ID")) + .put("MRole", Pair.of("ROLES", "ROLE_ID")) + .put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID")) + .put("MTable", Pair.of("TBLS","TBL_ID")) + .put("MStorageDescriptor", Pair.of("SDS", "SD_ID")) + .put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID")) + .put("MColumnDescriptor", Pair.of("CDS", "CD_ID")) + .put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID")) + .put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID")) + .put("MPartition", Pair.of("PARTITIONS", "PART_ID")) + .put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID")) + .put("MFunction", Pair.of("FUNCS", "FUNC_ID")) + .put("MIndex", Pair.of("IDXS", "INDEX_ID")) + .put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID")) + .build(); + + System.out.println("Validating sequence number for SEQUENCE_TABLE"); + + boolean isValid = true; + try { + Statement stmt = conn.createStatement(); + for (Map.Entry<String, Pair<String, String>> e : seqNameToTable.entrySet()) { + String tableName = e.getValue().getLeft(); + String tableKey = e.getValue().getRight(); + String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + e.getKey(); + String seqQuery = schemaTool.quote(QUERY_SEQ); + String maxIdQuery = String.format(schemaTool.quote(QUERY_MAX_ID), tableKey, tableName); + + ResultSet res = stmt.executeQuery(maxIdQuery); + if (res.next()) { + long maxId = res.getLong(1); + if (maxId > 0) { + PreparedStatement stmtSeq = conn.prepareStatement(seqQuery); + stmtSeq.setString(1, fullSequenceName); + ResultSet resSeq = stmtSeq.executeQuery(); + if (!resSeq.next()) { + isValid = false; + System.err.println("Missing SEQUENCE_NAME " + e.getKey() + " from SEQUENCE_TABLE"); + } else if (resSeq.getLong(1) < maxId) { + isValid = false; + System.err.println("NEXT_VAL for " + e.getKey() + " in SEQUENCE_TABLE < max(" + tableKey + + ") in " + tableName); + } + } + } + } + + System.out.println(isValid ? "[SUCCESS]\n" :"[FAIL]\n"); + return isValid; + } catch (SQLException e) { + throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e); + } + } + + @VisibleForTesting + boolean validateSchemaTables(Connection conn) throws HiveMetaException { + System.out.println("Validating metastore schema tables"); + String version = null; + try { + MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false); + version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); + } catch (HiveMetaException he) { + System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage()); + System.out.println("Failed in schema table validation."); + LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage(), he); + return false; + } + + Connection hmsConn = schemaTool.getConnectionToMetastore(false); + + LOG.debug("Validating tables in the schema for version " + version); + List<String> dbTables = new ArrayList<>(); + ResultSet rs = null; + try { + String schema = null; + try { + schema = hmsConn.getSchema(); + } catch (SQLFeatureNotSupportedException e) { + LOG.debug("schema is not supported"); + } + + DatabaseMetaData metadata = conn.getMetaData(); + rs = metadata.getTables(null, schema, "%", new String[] {"TABLE"}); + + while (rs.next()) { + String table = rs.getString("TABLE_NAME"); + dbTables.add(table.toLowerCase()); + LOG.debug("Found table " + table + " in HMS dbstore"); + } + } catch (SQLException e) { + throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + + e.getMessage(), e); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + throw new HiveMetaException("Failed to close resultset", e); + } + } + } + + // parse the schema file to determine the tables that are expected to exist + // we are using oracle schema because it is simpler to parse, no quotes or backticks etc + List<String> schemaTables = new ArrayList<>(); + List<String> subScripts = new ArrayList<>(); + + String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent(); + String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(), + schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath(); + try { + LOG.debug("Parsing schema script " + schemaFile); + subScripts.addAll(findCreateTable(schemaFile, schemaTables)); + while (subScripts.size() > 0) { + schemaFile = baseDir + "/" + schemaTool.getDbType() + "/" + subScripts.remove(0); + LOG.debug("Parsing subscript " + schemaFile); + subScripts.addAll(findCreateTable(schemaFile, schemaTables)); + } + } catch (Exception e) { + System.err.println("Exception in parsing schema file. Cause:" + e.getMessage()); + System.out.println("Failed in schema table validation."); + return false; + } + + LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]"); + LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]"); + + // now diff the lists + schemaTables.removeAll(dbTables); + if (schemaTables.size() > 0) { + Collections.sort(schemaTables); + System.err.println("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] " + + "are missing from the metastore database schema."); + System.out.println("[FAIL]\n"); + return false; + } else { + System.out.println("[SUCCESS]\n"); + return true; + } + } + + @VisibleForTesting + List<String> findCreateTable(String path, List<String> tableList) throws Exception { + if (!(new File(path)).exists()) { + throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table"); + } + + List<String> subs = new ArrayList<>(); + NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(schemaTool.getDbType(), false); + Pattern regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*"); + + try (BufferedReader reader = new BufferedReader(new FileReader(path))) { + String line = null; + while ((line = reader.readLine()) != null) { + if (sp.isNestedScript(line)) { + String subScript = sp.getScriptName(line); + LOG.debug("Schema subscript " + subScript + " found"); + subs.add(subScript); + continue; + } + line = line.replaceAll("( )+", " "); //suppress multi-spaces + line = line.replaceAll("\\(", " "); + line = line.replaceAll("IF NOT EXISTS ", ""); + line = line.replaceAll("`", ""); + line = line.replaceAll("'", ""); + line = line.replaceAll("\"", ""); + Matcher matcher = regexp.matcher(line); + + if (matcher.find()) { + String table = matcher.group(2); + if (schemaTool.getDbType().equals("derby")) { + table = table.replaceAll("APP\\.", ""); + } + tableList.add(table.toLowerCase()); + LOG.debug("Found table " + table + " in the schema"); + } + } + } catch (IOException ex){ + throw new Exception(ex.getMessage()); + } + + return subs; + } + + @VisibleForTesting + boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException { + System.out.println("Validating DFS locations"); + boolean rtn = true; + rtn &= checkMetaStoreDBLocation(conn, defaultServers); + rtn &= checkMetaStoreTableLocation(conn, defaultServers); + rtn &= checkMetaStorePartitionLocation(conn, defaultServers); + rtn &= checkMetaStoreSkewedColumnsLocation(conn, defaultServers); + System.out.println(rtn ? "[SUCCESS]\n" : "[FAIL]\n"); + return rtn; + } + + private static final String QUERY_DB_LOCATION = + " select dbt.<q>DB_ID<q>, " + + " dbt.<q>NAME<q>, " + + " dbt.<q>DB_LOCATION_URI<q> " + + " from <q>DBS<q> dbt " + + "order by dbt.<q>DB_ID<q> "; + + private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException { + String dbLocQuery = schemaTool.quote(QUERY_DB_LOCATION); + + int numOfInvalid = 0; + try (Statement stmt = conn.createStatement(); + ResultSet res = stmt.executeQuery(dbLocQuery)) { + while (res.next()) { + String locValue = res.getString(3); + String dbName = getNameOrID(res, 2, 1); + if (!checkLocation("Database " + dbName, locValue, defaultServers)) { + numOfInvalid++; + } + } + } catch (SQLException e) { + throw new HiveMetaException("Failed to get DB Location Info.", e); + } + return numOfInvalid == 0; + } + + private static final String TAB_ID_RANGE_QUERY = + "select max(<q>TBL_ID<q>), " + + " min(<q>TBL_ID<q>) " + + " from <q>TBLS<q> "; + + private static final String TAB_LOC_QUERY = + " select tbl.<q>TBL_ID<q>, " + + " tbl.<q>TBL_NAME<q>, " + + " sd.<q>LOCATION<q>, " + + " dbt.<q>DB_ID<q>, " + + " dbt.<q>NAME<q> " + + " from <q>TBLS<q> tbl " + + "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = tbl.<q>SD_ID<q> " + + "inner join <q>DBS<q> dbt on tbl.<q>DB_ID<q> = dbt.<q>DB_ID<q> " + + " where tbl.<q>TBL_TYPE<q> != '%s' " + + " and tbl.<q>TBL_ID<q> >= ? " + + " and tbl.<q>TBL_ID<q> <= ? " + + " order by tbl.<q>TBL_ID<q> "; + + private static final int TAB_LOC_CHECK_SIZE = 2000; + + private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers) + throws HiveMetaException { + String tabIDRangeQuery = schemaTool.quote(TAB_ID_RANGE_QUERY); + String tabLocQuery = String.format(schemaTool.quote(TAB_LOC_QUERY), TableType.VIRTUAL_VIEW); + + try { + long maxID = 0, minID = 0; + try (Statement stmt = conn.createStatement(); + ResultSet res = stmt.executeQuery(tabIDRangeQuery)) { + if (res.next()) { + maxID = res.getLong(1); + minID = res.getLong(2); + } + } + + int numOfInvalid = 0; + try (PreparedStatement pStmt = conn.prepareStatement(tabLocQuery)) { + while (minID <= maxID) { + pStmt.setLong(1, minID); + pStmt.setLong(2, minID + TAB_LOC_CHECK_SIZE); + try (ResultSet res = pStmt.executeQuery()) { + while (res.next()) { + String locValue = res.getString(3); + String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1); + if (!checkLocation(entity, locValue, defaultServers)) { + numOfInvalid++; + } + } + } + minID += TAB_LOC_CHECK_SIZE + 1; + } + } + + return numOfInvalid == 0; + } catch (SQLException e) { + throw new HiveMetaException("Failed to get Table Location Info.", e); + } + } + + private static final String QUERY_PART_ID_RANGE = + "select max(<q>PART_ID<q>)," + + " min(<q>PART_ID<q>)" + + " from <q>PARTITIONS<q> "; + + private static final String QUERY_PART_LOC = + " select pt.<q>PART_ID<q>, " + + " pt.<q>PART_NAME<q>, " + + " sd.<q>LOCATION<q>, " + + " tbl.<q>TBL_ID<q>, " + + " tbl.<q>TBL_NAME<q>, " + + " dbt.<q>DB_ID<q>, " + + " dbt.<q>NAME<q> " + + " from <q>PARTITIONS<q> pt " + + "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = pt.<q>SD_ID<q> " + + "inner join <q>TBLS<q> tbl on tbl.<q>TBL_ID<q> = pt.<q>TBL_ID<q> " + + "inner join <q>DBS<q> dbt on dbt.<q>DB_ID<q> = tbl.<q>DB_ID<q> " + + " where pt.<q>PART_ID<q> >= ? " + + " and pt.<q>PART_ID<q> <= ? " + + " order by tbl.<q>TBL_ID<q> "; + + private static final int PART_LOC_CHECK_SIZE = 2000; + + private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultServers) + throws HiveMetaException { + String queryPartIDRange = schemaTool.quote(QUERY_PART_ID_RANGE); + String queryPartLoc = schemaTool.quote(QUERY_PART_LOC); + + try { + long maxID = 0, minID = 0; + try (Statement stmt = conn.createStatement(); + ResultSet res = stmt.executeQuery(queryPartIDRange)) { + if (res.next()) { + maxID = res.getLong(1); + minID = res.getLong(2); + } + } + + int numOfInvalid = 0; + try (PreparedStatement pStmt = conn.prepareStatement(queryPartLoc)) { + while (minID <= maxID) { + pStmt.setLong(1, minID); + pStmt.setLong(2, minID + PART_LOC_CHECK_SIZE); + try (ResultSet res = pStmt.executeQuery()) { + while (res.next()) { + String locValue = res.getString(3); + String entity = "Database " + getNameOrID(res, 7, 6) + ", Table " + getNameOrID(res, 5, 4) + + ", Partition " + getNameOrID(res, 2, 1); + if (!checkLocation(entity, locValue, defaultServers)) { + numOfInvalid++; + } + } + } + minID += PART_LOC_CHECK_SIZE + 1; + } + } + + return numOfInvalid == 0; + } catch (SQLException e) { + throw new HiveMetaException("Failed to get Partition Location Info.", e); + } + } + + private static final String QUERY_SKEWED_COL_ID_RANGE = + "select max(<q>STRING_LIST_ID_KID<q>), " + + " min(<q>STRING_LIST_ID_KID<q>) " + + " from <q>SKEWED_COL_VALUE_LOC_MAP<q> "; + + private static final String QUERY_SKEWED_COL_LOC = + " select t.<q>TBL_NAME<q>, " + + " t.<q>TBL_ID<q>, " + + " sk.<q>STRING_LIST_ID_KID<q>, " + + " sk.<q>LOCATION<q>, " + + " db.<q>NAME<q>, " + + " db.<q>DB_ID<q> " + + " from <q>TBLS<q> t " + + " join <q>SDS<q> s on s.<q>SD_ID<q> = t.<q>SD_ID<q> " + + " join <q>DBS<q> db on db.<q>DB_ID<q> = t.<q>DB_ID<q> " + + " join <q>SKEWED_COL_VALUE_LOC_MAP<q> sk on sk.<q>SD_ID<q> = s.<q>SD_ID<q> " + + " where sk.<q>STRING_LIST_ID_KID<q> >= ? " + + " and sk.<q>STRING_LIST_ID_KID<q> <= ? " + + "order by t.<q>TBL_ID<q> "; + + private static final int SKEWED_COL_LOC_CHECK_SIZE = 2000; + + private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers) + throws HiveMetaException { + String querySkewedColIDRange = schemaTool.quote(QUERY_SKEWED_COL_ID_RANGE); + String querySkewedColLoc = schemaTool.quote(QUERY_SKEWED_COL_LOC); + + try { + long maxID = 0, minID = 0; + try (Statement stmt = conn.createStatement(); + ResultSet res = stmt.executeQuery(querySkewedColIDRange)) { + if (res.next()) { + maxID = res.getLong(1); + minID = res.getLong(2); + } + } + + int numOfInvalid = 0; + try (PreparedStatement pStmt = conn.prepareStatement(querySkewedColLoc)) { + while (minID <= maxID) { + pStmt.setLong(1, minID); + pStmt.setLong(2, minID + SKEWED_COL_LOC_CHECK_SIZE); + try (ResultSet res = pStmt.executeQuery()) { + while (res.next()) { + String locValue = res.getString(4); + String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) + + ", String list " + res.getString(3); + if (!checkLocation(entity, locValue, defaultServers)) { + numOfInvalid++; + } + } + } + minID += SKEWED_COL_LOC_CHECK_SIZE + 1; + } + } + + return numOfInvalid == 0; + } catch (SQLException e) { + throw new HiveMetaException("Failed to get skewed columns location info.", e); + } + } + + /** + * Check if the location is valid for the given entity. + * @param entity the entity to represent a database, partition or table + * @param entityLocation the location + * @param defaultServers a list of the servers that the location needs to match. + * The location host needs to match one of the given servers. + * If empty, then no check against such list. + * @return true if the location is valid + */ + private boolean checkLocation(String entity, String entityLocation, URI[] defaultServers) { + boolean isValid = true; + + if (entityLocation == null) { + System.err.println(entity + ", Error: empty location"); + isValid = false; + } else { + try { + URI currentUri = new Path(entityLocation).toUri(); + String scheme = currentUri.getScheme(); + String path = currentUri.getPath(); + if (StringUtils.isEmpty(scheme)) { + System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location scheme."); + isValid = false; + } else if (StringUtils.isEmpty(path)) { + System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location path."); + isValid = false; + } else if (ArrayUtils.isNotEmpty(defaultServers) && currentUri.getAuthority() != null) { + String authority = currentUri.getAuthority(); + boolean matchServer = false; + for(URI server : defaultServers) { + if (StringUtils.equalsIgnoreCase(server.getScheme(), scheme) && + StringUtils.equalsIgnoreCase(server.getAuthority(), authority)) { + matchServer = true; + break; + } + } + if (!matchServer) { + System.err.println(entity + ", Location: " + entityLocation + ", Error: mismatched server."); + isValid = false; + } + } + + // if there is no path element other than "/", report it but not fail + if (isValid && StringUtils.containsOnly(path, "/")) { + System.err.println(entity + ", Location: "+ entityLocation + ", Warn: location set to root, " + + "not a recommended config."); + } + } catch (Exception pe) { + System.err.println(entity + ", Error: invalid location - " + pe.getMessage()); + isValid =false; + } + } + + return isValid; + } + + private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLException { + String itemName = res.getString(nameInx); + return (itemName == null || itemName.isEmpty()) ? "ID: " + res.getString(idInx) : "Name: " + itemName; + } + + private static final String QUERY_COLUMN_NULL_VALUES = + " select t.*" + + " from <q>TBLS<q> t" + + " where t.<q>SD_ID<q> IS NULL" + + " and (t.<q>TBL_TYPE<q> = '" + TableType.EXTERNAL_TABLE + "' or" + + " t.<q>TBL_TYPE<q> = '" + TableType.MANAGED_TABLE + "') " + + "order by t.<q>TBL_ID<q> "; + + @VisibleForTesting + boolean validateColumnNullValues(Connection conn) throws HiveMetaException { + System.out.println("Validating columns for incorrect NULL values."); + + boolean isValid = true; + String queryColumnNullValues = schemaTool.quote(QUERY_COLUMN_NULL_VALUES); + + try (Statement stmt = conn.createStatement(); + ResultSet res = stmt.executeQuery(queryColumnNullValues)) { + while (res.next()) { + long tableId = res.getLong("TBL_ID"); + String tableName = res.getString("TBL_NAME"); + String tableType = res.getString("TBL_TYPE"); + isValid = false; + System.err.println("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType); + } + + System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n"); + return isValid; + } catch(SQLException e) { + throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java index 4722a56..2915720 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.metastore.dbinstall; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.HiveMetaException; -import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool; import org.junit.After; import org.junit.Assert; @@ -46,8 +44,6 @@ public abstract class DbInstallBase { private static final String FIRST_VERSION = "1.2.0"; private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000; - private String metastoreHome; - protected abstract String getDockerContainerName(); protected abstract String getDockerImageName(); protected abstract String[] getDockerAdditionalArgs(); @@ -86,7 +82,7 @@ public abstract class DbInstallBase { throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 + " seconds"); } - MetastoreSchemaTool.homeDir = metastoreHome = System.getProperty("test.tmp.dir", "target/tmp"); + MetastoreSchemaTool.setHomeDirForTesting(); } @After @@ -145,7 +141,7 @@ public abstract class DbInstallBase { } private int createUser() { - return MetastoreSchemaTool.run(buildArray( + return new MetastoreSchemaTool().run(buildArray( "-createUser", "-dbType", getDbType(), @@ -167,7 +163,7 @@ public abstract class DbInstallBase { } private int installLatest() { - return MetastoreSchemaTool.run(buildArray( + return new MetastoreSchemaTool().run(buildArray( "-initSchema", "-dbType", getDbType(), @@ -183,7 +179,7 @@ public abstract class DbInstallBase { } private int installAVersion(String version) { - return MetastoreSchemaTool.run(buildArray( + return new MetastoreSchemaTool().run(buildArray( "-initSchemaTo", version, "-dbType", @@ -200,7 +196,7 @@ public abstract class DbInstallBase { } private int upgradeToLatest() { - return MetastoreSchemaTool.run(buildArray( + return new MetastoreSchemaTool().run(buildArray( "-upgradeSchema", "-dbType", getDbType(), @@ -219,17 +215,6 @@ public abstract class DbInstallBase { return strs; } - private String getCurrentVersionMinusOne() throws HiveMetaException { - List<String> scripts = MetaStoreSchemaInfoFactory.get( - MetastoreConf.newMetastoreConf(), metastoreHome, getDbType() - ).getUpgradeScripts(FIRST_VERSION); - Assert.assertTrue(scripts.size() > 0); - String lastUpgradePath = scripts.get(scripts.size() - 1); - String version = lastUpgradePath.split("-")[1]; - LOG.info("Current version minus 1 is " + version); - return version; - } - @Test public void install() { Assert.assertEquals(0, createUser()); http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java index 9e425cf..0657ae5 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java @@ -27,10 +27,13 @@ import java.io.OutputStream; import java.io.PrintStream; import java.net.URI; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.Random; +import org.apache.commons.dbcp.DelegatingConnection; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.text.StrTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; @@ -56,6 +59,8 @@ public class TestSchemaToolForMetastore { private String testMetastoreDB; private PrintStream errStream; private PrintStream outStream; + private String argsBase; + private SchemaToolTaskValidate validator; @Before public void setUp() throws HiveMetaException, IOException { @@ -64,14 +69,21 @@ public class TestSchemaToolForMetastore { System.setProperty(ConfVars.CONNECT_URL_KEY.toString(), "jdbc:derby:" + testMetastoreDB + ";create=true"); conf = MetastoreConf.newMetastoreConf(); - schemaTool = new MetastoreSchemaTool( - System.getProperty("test.tmp.dir", "target/tmp"), conf, "derby"); - schemaTool.setUserName(MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME)); - schemaTool.setPassWord(MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD)); + schemaTool = new MetastoreSchemaTool(); + schemaTool.init(System.getProperty("test.tmp.dir", "target/tmp"), + new String[]{"-dbType", "derby", "--info"}, null, conf); + String userName = MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME); + String passWord = MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD); + schemaTool.setUserName(userName); + schemaTool.setPassWord(passWord); + argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " "; System.setProperty("beeLine.system.exit", "true"); errStream = System.err; outStream = System.out; conn = schemaTool.getConnectionToMetastore(false); + + validator = new SchemaToolTaskValidate(); + validator.setHiveSchemaTool(schemaTool); } @After @@ -87,24 +99,26 @@ public class TestSchemaToolForMetastore { } } - // Test the sequence validation functionality + /* + * Test the sequence validation functionality + */ @Test public void testValidateSequences() throws Exception { - schemaTool.doInit(); + execute(new SchemaToolTaskInit(), "-initSchema"); // Test empty database - boolean isValid = schemaTool.validateSequences(conn); + boolean isValid = validator.validateSequences(conn); Assert.assertTrue(isValid); // Test valid case String[] scripts = new String[] { + "insert into CTLGS values(99, 'test_cat_1', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');", "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);", - "insert into CTLGS values(37, 'mycat', 'my description', 'hdfs://tmp');", - "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');" + "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');" }; File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSequences(conn); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSequences(conn); Assert.assertTrue(isValid); // Test invalid case @@ -112,25 +126,27 @@ public class TestSchemaToolForMetastore { "delete from SEQUENCE_TABLE;", "delete from DBS;", "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);", - "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');" + "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');" }; scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSequences(conn); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSequences(conn); Assert.assertFalse(isValid); } - // Test to validate that all tables exist in the HMS metastore. + /* + * Test to validate that all tables exist in the HMS metastore. + */ @Test public void testValidateSchemaTables() throws Exception { - schemaTool.doInit("1.2.0"); + execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0"); - boolean isValid = schemaTool.validateSchemaTables(conn); + boolean isValid = validator.validateSchemaTables(conn); Assert.assertTrue(isValid); - // upgrade from 2.0.0 schema and re-validate - schemaTool.doUpgrade("1.2.0"); - isValid = schemaTool.validateSchemaTables(conn); + // upgrade from 1.2.0 schema and re-validate + execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0"); + isValid = validator.validateSchemaTables(conn); Assert.assertTrue(isValid); // Simulate a missing table scenario by renaming a couple of tables @@ -140,8 +156,8 @@ public class TestSchemaToolForMetastore { }; File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaTables(conn); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSchemaTables(conn); Assert.assertFalse(isValid); // Restored the renamed tables @@ -151,31 +167,49 @@ public class TestSchemaToolForMetastore { }; scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaTables(conn); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSchemaTables(conn); Assert.assertTrue(isValid); - } + + // Check that an exception from getMetaData() is reported correctly + try { + // Make a Connection object that will throw an exception + BadMetaDataConnection bad = new BadMetaDataConnection(conn); + validator.validateSchemaTables(bad); + Assert.fail("did not get expected exception"); + } catch (HiveMetaException hme) { + String message = hme.getMessage(); + Assert.assertTrue("Bad HiveMetaException message :" + message, + message.contains("Failed to retrieve schema tables from Hive Metastore DB")); + Throwable cause = hme.getCause(); + Assert.assertNotNull("HiveMetaException did not contain a cause", cause); + String causeMessage = cause.getMessage(); + Assert.assertTrue("Bad SQLException message: " + causeMessage, causeMessage.contains( + BadMetaDataConnection.FAILURE_TEXT)); + } + } + // Test the validation of incorrect NULL values in the tables @Test public void testValidateNullValues() throws Exception { - schemaTool.doInit(); + execute(new SchemaToolTaskInit(), "-initSchema"); // Test empty database - boolean isValid = schemaTool.validateColumnNullValues(conn); + boolean isValid = validator.validateColumnNullValues(conn); Assert.assertTrue(isValid); // Test valid case createTestHiveTableSchemas(); - isValid = schemaTool.validateColumnNullValues(conn); + isValid = validator.validateColumnNullValues(conn); // Test invalid case String[] scripts = new String[] { "update TBLS set SD_ID=null" }; File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateColumnNullValues(conn); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateColumnNullValues(conn); Assert.assertFalse(isValid); } @@ -183,7 +217,7 @@ public class TestSchemaToolForMetastore { @Test public void testSchemaInitDryRun() throws Exception { schemaTool.setDryRun(true); - schemaTool.doInit("3.0.0"); + execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0"); schemaTool.setDryRun(false); try { schemaTool.verifySchemaVersion(); @@ -197,10 +231,10 @@ public class TestSchemaToolForMetastore { // Test dryrun of schema upgrade @Test public void testSchemaUpgradeDryRun() throws Exception { - schemaTool.doInit("1.2.0"); + execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0"); schemaTool.setDryRun(true); - schemaTool.doUpgrade("1.2.0"); + execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0"); schemaTool.setDryRun(false); try { schemaTool.verifySchemaVersion(); @@ -218,8 +252,7 @@ public class TestSchemaToolForMetastore { public void testSchemaInit() throws Exception { IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf, System.getProperty("test.tmp.dir", "target/tmp"), "derby"); - LOG.info("Starting testSchemaInit"); - schemaTool.doInit(metastoreSchemaInfo.getHiveSchemaVersion()); + execute(new SchemaToolTaskInit(), "-initSchemaTo " + metastoreSchemaInfo.getHiveSchemaVersion()); schemaTool.verifySchemaVersion(); } @@ -227,35 +260,35 @@ public class TestSchemaToolForMetastore { * Test validation for schema versions */ @Test - public void testValidateSchemaVersions() throws Exception { - schemaTool.doInit(); - boolean isValid = schemaTool.validateSchemaVersions(); - // Test an invalid case with multiple versions - String[] scripts = new String[] { - "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')" - }; - File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); - Assert.assertFalse(isValid); - - scripts = new String[] { - "delete from VERSION where VER_ID = 100" - }; - scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); - Assert.assertTrue(isValid); - - // Test an invalid case without version - scripts = new String[] { - "delete from VERSION" - }; - scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); - Assert.assertFalse(isValid); - } + public void testValidateSchemaVersions() throws Exception { + execute(new SchemaToolTaskInit(), "-initSchema"); + boolean isValid = validator.validateSchemaVersions(); + // Test an invalid case with multiple versions + String[] scripts = new String[] { + "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')" + }; + File scriptFile = generateTestScript(scripts); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSchemaVersions(); + Assert.assertFalse(isValid); + + scripts = new String[] { + "delete from VERSION where VER_ID = 100" + }; + scriptFile = generateTestScript(scripts); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSchemaVersions(); + Assert.assertTrue(isValid); + + // Test an invalid case without version + scripts = new String[] { + "delete from VERSION" + }; + scriptFile = generateTestScript(scripts); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateSchemaVersions(); + Assert.assertFalse(isValid); + } /** * Test schema upgrade @@ -264,7 +297,7 @@ public class TestSchemaToolForMetastore { public void testSchemaUpgrade() throws Exception { boolean foundException = false; // Initialize 1.2.0 schema - schemaTool.doInit("1.2.0"); + execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0"); // verify that driver fails due to older version schema try { schemaTool.verifySchemaVersion(); @@ -298,17 +331,7 @@ public class TestSchemaToolForMetastore { System.setOut(outPrintStream); // Upgrade schema from 0.7.0 to latest - Exception caught = null; - try { - schemaTool.doUpgrade("1.2.0"); - } catch (Exception e) { - caught = e; - } - - LOG.info("stdout is " + stdout.toString()); - LOG.info("stderr is " + stderr.toString()); - - if (caught != null) Assert.fail(caught.getMessage()); + execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0"); // Verify that the schemaTool ran pre-upgrade scripts and ignored errors Assert.assertTrue(stderr.toString().contains(invalidPreUpgradeScript)); @@ -327,38 +350,38 @@ public class TestSchemaToolForMetastore { */ @Test public void testValidateLocations() throws Exception { - schemaTool.doInit(); + execute(new SchemaToolTaskInit(), "-initSchema"); URI defaultRoot = new URI("hdfs://myhost.com:8020"); URI defaultRoot2 = new URI("s3://myhost2.com:8888"); //check empty DB - boolean isValid = schemaTool.validateLocations(conn, null); + boolean isValid = validator.validateLocations(conn, null); Assert.assertTrue(isValid); - isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot,defaultRoot2}); + isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2}); Assert.assertTrue(isValid); - // Test valid case + // Test valid case String[] scripts = new String[] { - "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');", - "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", - "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');", - "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');", - "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);", - "insert into SKEWED_STRING_LIST values(1);", - "insert into SKEWED_STRING_LIST values(2);", - "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');", - "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');" - }; + "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');", + "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');", + "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2');", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", + "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');", + "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');", + "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);", + "insert into SKEWED_STRING_LIST values(1);", + "insert into SKEWED_STRING_LIST values(2);", + "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');", + "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');" + }; File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateLocations(conn, null); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateLocations(conn, null); Assert.assertTrue(isValid); - isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2}); + isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2}); Assert.assertTrue(isValid); scripts = new String[] { "delete from SKEWED_COL_VALUE_LOC_MAP;", @@ -367,10 +390,10 @@ public class TestSchemaToolForMetastore { "delete from TBLS;", "delete from SDS;", "delete from DBS;", - "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');", - "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');", - "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');", + "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');", + "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2');", + "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2');", + "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2');", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", @@ -398,23 +421,23 @@ public class TestSchemaToolForMetastore { "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'file:///user/admin/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');" }; scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); - isValid = schemaTool.validateLocations(conn, null); + schemaTool.execSql(scriptFile.getPath()); + isValid = validator.validateLocations(conn, null); Assert.assertFalse(isValid); - isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2}); + isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2}); Assert.assertFalse(isValid); } @Test public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException { - schemaTool.doInit("3.0.0"); + execute(new SchemaToolTaskInit(), "-initSchemaTo 3.0.0"); validateMetastoreDbPropertiesTable(); } @Test public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException { - schemaTool.doInit("1.2.0"); - schemaTool.doUpgrade(); + execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0"); + execute(new SchemaToolTaskUpgrade(), "-upgradeSchema"); validateMetastoreDbPropertiesTable(); } @@ -432,7 +455,7 @@ public class TestSchemaToolForMetastore { } private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException { - boolean isValid = schemaTool.validateSchemaTables(conn); + boolean isValid = (boolean) validator.validateSchemaTables(conn); Assert.assertTrue(isValid); // adding same property key twice should throw unique key constraint violation exception String[] scripts = new String[] { @@ -441,12 +464,13 @@ public class TestSchemaToolForMetastore { File scriptFile = generateTestScript(scripts); Exception ex = null; try { - schemaTool.runSqlLine(scriptFile.getPath()); + schemaTool.execSql(scriptFile.getPath()); } catch (Exception iox) { ex = iox; } Assert.assertTrue(ex != null && ex instanceof IOException); } + /** * Write out a dummy pre-upgrade script with given SQL statement. */ @@ -476,6 +500,35 @@ public class TestSchemaToolForMetastore { "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);" }; File scriptFile = generateTestScript(scripts); - schemaTool.runSqlLine(scriptFile.getPath()); + schemaTool.execSql(scriptFile.getPath()); + } + + /** + * A mock Connection class that throws an exception out of getMetaData(). + */ + class BadMetaDataConnection extends DelegatingConnection { + static final String FAILURE_TEXT = "fault injected"; + + BadMetaDataConnection(Connection connection) { + super(connection); + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + throw new SQLException(FAILURE_TEXT); + } + } + + private void execute(SchemaToolTask task, String taskArgs) throws HiveMetaException { + try { + StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"'); + SchemaToolCommandLine cl = new SchemaToolCommandLine(tokenizer.getTokenArray(), null); + task.setCommandLineArguments(cl); + } catch (Exception e) { + throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e); + } + + task.setHiveSchemaTool(schemaTool); + task.execute(); } }