This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 82feb02 HIVE-21034: Add option to schematool to drop Hive databases (Daniel Voros via Ashutosh Chauhan) 82feb02 is described below commit 82feb0265f291065e6c7b26695c984bf9cb63cde Author: Daniel Voros <daniel.vo...@gmail.com> AuthorDate: Mon Apr 1 12:19:45 2019 +0200 HIVE-21034: Add option to schematool to drop Hive databases (Daniel Voros via Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> --- .../tools/schematool/HiveSchemaHelper.java | 2 +- .../tools/schematool/MetastoreSchemaTool.java | 14 ++- .../tools/schematool/SchemaToolCommandLine.java | 13 +++ .../tools/schematool/SchemaToolTaskDrop.java | 130 +++++++++++++++++++++ .../tools/schematool/TestSchemaToolTaskDrop.java | 129 ++++++++++++++++++++ 5 files changed, 282 insertions(+), 6 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java index caf9c31..cfec036 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java @@ -71,7 +71,7 @@ public class HiveSchemaHelper { driver = driver == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver; if (printInfo) { logAndPrintToStdout("Metastore connection URL:\t " + url); - logAndPrintToStdout("Metastore Connection Driver :\t " + driver); + logAndPrintToStdout("Metastore connection Driver :\t " + driver); logAndPrintToStdout("Metastore connection User:\t " + userName); if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) { logAndPrintToStdout("Metastore connection Password:\t " + password); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java index 1517fa1..eafe0c6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java @@ -111,7 +111,7 @@ public class MetastoreSchemaTool { this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf, metastoreHome, dbType); // If the dbType is "hive", this is setting up the information schema in Hive. // We will set the default jdbc url and driver. - // It is overriden by command line options if passed (-url and -driver + // It is overridden by command line options if passed (-url and -driver) if (dbType.equalsIgnoreCase(HiveSchemaHelper.DB_HIVE)) { this.url = HiveSchemaHelper.EMBEDDED_HS2_URL; this.driver = HiveSchemaHelper.HIVE_JDBC_DRIVER; @@ -437,6 +437,8 @@ public class MetastoreSchemaTool { task = new SchemaToolTaskMoveTable(); } else if (cmdLine.hasOption("createUser")) { task = new SchemaToolTaskCreateUser(); + } else if (cmdLine.hasOption("dropAllDatabases")) { + task = new SchemaToolTaskDrop(); } else { throw new HiveMetaException("No task defined!"); } @@ -456,10 +458,12 @@ public class MetastoreSchemaTool { logAndPrintToError("SQL Error code: " + ((SQLException) t).getErrorCode()); } } - if (cmdLine.hasOption("verbose")) { - e.printStackTrace(); - } else { - logAndPrintToError("Use --verbose for detailed stacktrace."); + if (cmdLine != null) { + if (cmdLine.hasOption("verbose")) { + e.printStackTrace(); + } else { + logAndPrintToError("Use --verbose for detailed stacktrace."); + } } logAndPrintToError("*** schemaTool failed ***"); return 1; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java index 84b85e9..6282078 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java @@ -47,6 +47,9 @@ public class SchemaToolCommandLine { .withDescription("Schema initialization to a version") .create("initSchemaTo"); Option initOrUpgradeSchemaOpt = new Option("initOrUpgradeSchema", "Initialize or upgrade schema to latest version"); + Option dropDbOpt = new Option("dropAllDatabases", "Drop all Hive databases (with CASCADE). " + + "This will remove all managed data!"); + Option yesOpt = new Option("yes", "Don't ask for confirmation when using -dropAllDatabases."); Option validateOpt = new Option("validate", "Validate the database"); Option createCatalog = OptionBuilder .hasArg() @@ -77,6 +80,7 @@ public class SchemaToolCommandLine { .addOption(upgradeOpt) .addOption(upgradeFromOpt) .addOption(initOpt) + .addOption(dropDbOpt) .addOption(initToOpt) .addOption(initOrUpgradeSchemaOpt) .addOption(validateOpt) @@ -185,6 +189,7 @@ public class SchemaToolCommandLine { options.addOption(hiveUserOpt); options.addOption(hivePasswdOpt); options.addOption(hiveDbOpt); + options.addOption(yesOpt); if (additionalOptions != null) options.addOptionGroup(additionalOptions); return options; @@ -276,6 +281,14 @@ public class SchemaToolCommandLine { (cl.hasOption("fromDatabase") || cl.hasOption("toDatabase"))) { printAndExit("fromDatabase and toDatabase may be set only for moveTable"); } + + if (cl.hasOption("dropAllDatabases") && !HiveSchemaHelper.DB_HIVE.equals(dbType)) { + printAndExit("dropAllDatabases can only be used with dbType=hive"); + } + + if (cl.hasOption("yes") && !cl.hasOption("dropAllDatabases")) { + printAndExit("yes can only be used with dropAllDatabases"); + } } private void printAndExit(String reason) throws ParseException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java new file mode 100644 index 0000000..aa8ca94 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools.schematool; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.Warehouse; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +/** + * {@link SchemaToolTaskDrop} drops all data from Hive. It invokes DROP TABLE on all + * tables of the default database and DROP DATABASE CASCADE on all other databases. + */ +public class SchemaToolTaskDrop extends SchemaToolTask { + + @VisibleForTesting + boolean yes = false; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cmdLine) { + if (cmdLine.hasOption("yes")) { + this.yes = true; + } + } + + @Override + void execute() throws HiveMetaException { + // Need to confirm unless it's a dry run or specified -yes + if (!schemaTool.isDryRun() && !this.yes) { + boolean confirmed = promptToConfirm(); + if (!confirmed) { + System.out.println("Operation cancelled, exiting."); + return; + } + } + + Connection conn = schemaTool.getConnectionToMetastore(true); + try { + try (Statement stmt = conn.createStatement()) { + final String def = Warehouse.DEFAULT_DATABASE_NAME; + + // List databases + List<String> databases = new ArrayList<>(); + try (ResultSet rs = stmt.executeQuery("SHOW DATABASES")) { + while (rs.next()) { + databases.add(rs.getString(1)); + } + } + + // Drop databases + for (String database : databases) { + // Don't try to drop 'default' database as it's not allowed + if (!def.equalsIgnoreCase(database)) { + if (schemaTool.isDryRun()) { + System.out.println("would drop database " + database); + } else { + logIfVerbose("dropping database " + database); + stmt.execute(String.format("DROP DATABASE `%s` CASCADE", database)); + } + } + } + + // List tables in 'default' database + List<String> tables = new ArrayList<>(); + try (ResultSet rs = stmt.executeQuery(String.format("SHOW TABLES IN `%s`", def))) { + while (rs.next()) { + tables.add(rs.getString(1)); + } + } + + // Drop tables in 'default' database + for (String table : tables) { + if (schemaTool.isDryRun()) { + System.out.println("would drop table " + table); + } else { + logIfVerbose("dropping table " + table); + stmt.execute(String.format("DROP TABLE `%s`.`%s`", def, table)); + } + } + } + } catch (SQLException se) { + throw new HiveMetaException("Failed to drop databases.", se); + } + } + + /** + * Display "are you sure? y/n" on command line and return what the user has chosen + * @return + */ + private boolean promptToConfirm() { + System.out.print("This operation will delete ALL managed data in Hive. " + + "Are you sure you want to continue (y/[n])?"); + Scanner scanner = new Scanner(System.in, "UTF-8"); + String input = scanner.nextLine(); + + if ("y".equalsIgnoreCase(input) || "yes".equalsIgnoreCase(input)) { + return true; + } + return false; + } + + private void logIfVerbose(String msg) { + if (schemaTool.isVerbose()) { + System.out.println(msg); + } + } +} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java new file mode 100644 index 0000000..271f83a --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools.schematool; + +import org.apache.hadoop.hive.metastore.Warehouse; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +/** + * Testing SchemaToolTaskDrop. + */ +public class TestSchemaToolTaskDrop { + + private SchemaToolTaskDrop uut; + + private Statement stmtMock; + + private final InputStream stdin = System.in; + + @Before + public void setUp() throws Exception { + uut = new SchemaToolTaskDrop(); + uut.schemaTool = mock(MetastoreSchemaTool.class); + } + + @After + public void tearDown() throws Exception { + System.setIn(stdin); + } + + private void mockPromptWith(String answer) { + InputStream in = new ByteArrayInputStream((answer + "\n").getBytes()); + System.setIn(in); + } + + private void setUpTwoDatabases() throws Exception { + Connection connMock = mock(Connection.class); + stmtMock = mock(Statement.class); + + // Return two databases: `mydb` and `default` + ResultSet databasesResult = mock(ResultSet.class); + when(databasesResult.next()).thenReturn(true, true, false); + when(databasesResult.getString(anyInt())).thenReturn("mydb", Warehouse.DEFAULT_DATABASE_NAME); + + // Return two tables: `table1` and `table2` + ResultSet tablesResult = mock(ResultSet.class); + when(tablesResult.next()).thenReturn(true, true, false); + when(tablesResult.getString(anyInt())).thenReturn("table1", "table2"); + + when(stmtMock.executeQuery(anyString())).thenReturn(databasesResult, tablesResult); + when(connMock.createStatement()).thenReturn(stmtMock); + + when(uut.schemaTool.getConnectionToMetastore(anyBoolean())).thenReturn(connMock); + } + + @Test + public void testExecutePromptYes() throws Exception { + setUpTwoDatabases(); + mockPromptWith("y"); + + uut.execute(); + + Mockito.verify(stmtMock).execute("DROP DATABASE `mydb` CASCADE"); + Mockito.verify(stmtMock).execute(String.format("DROP TABLE `%s`.`table1`", Warehouse.DEFAULT_DATABASE_NAME)); + Mockito.verify(stmtMock).execute(String.format("DROP TABLE `%s`.`table2`", Warehouse.DEFAULT_DATABASE_NAME)); + Mockito.verify(stmtMock, times(3)).execute(anyString()); + } + + @Test + public void testExecutePromptNo() throws Exception { + setUpTwoDatabases(); + mockPromptWith("n"); + + uut.execute(); + + Mockito.verify(stmtMock, times(0)).execute(anyString()); + } + + @Test + public void testExecuteDryRun() throws Exception { + setUpTwoDatabases(); + when(uut.schemaTool.isDryRun()).thenReturn(true); + + uut.execute(); + + Mockito.verify(stmtMock, times(0)).execute(anyString()); + } + + @Test + public void testExecuteWithYes() throws Exception { + setUpTwoDatabases(); + uut.yes = true; + + uut.execute(); + + Mockito.verify(stmtMock, times(3)).execute(anyString()); + } +}