This is an automated email from the ASF dual-hosted git repository. szita pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d91cc0c HIVE-23142: HiveStrictManagedMigration fails with tables that have null location (Adam Szita, reviewed by Marta Kuczora) d91cc0c is described below commit d91cc0cd84b7d0ecc0f29d44b109b46e21194eec Author: Adam Szita <sz...@cloudera.com> AuthorDate: Tue Apr 7 14:31:40 2020 +0200 HIVE-23142: HiveStrictManagedMigration fails with tables that have null location (Adam Szita, reviewed by Marta Kuczora) --- .../hadoop/hive/ql/util/HiveStrictManagedMigration.java | 17 +++++++++++++---- .../hive/ql/util/TestHiveStrictManagedMigration.java | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index e2e5983..5e658a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -34,6 +34,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -80,6 +81,8 @@ import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE; public class HiveStrictManagedMigration { private static final Logger LOG = LoggerFactory.getLogger(HiveStrictManagedMigration.class); + @VisibleForTesting + static int RC = 0; public enum TableMigrationOption { NONE, // Do nothing @@ -194,6 +197,7 @@ public class HiveStrictManagedMigration { public static void main(String[] args) throws Exception { RunOptions runOptions; + RC = 0; try { Options opts = createOptions(); @@ -210,7 +214,6 @@ public class HiveStrictManagedMigration { throw new Exception("Error processing options", err); } - int rc = 0; HiveStrictManagedMigration migration = null; try { HiveConf conf = hiveConf == null ? new HiveConf() : hiveConf; @@ -227,7 +230,7 @@ public class HiveStrictManagedMigration { migration.run(); } catch (Exception err) { LOG.error("Failed with error", err); - rc = -1; + RC = -1; } finally { if (migration != null) { migration.cleanup(); @@ -236,7 +239,7 @@ public class HiveStrictManagedMigration { // TODO: Something is preventing the process from terminating after main(), adding exit() as hacky solution. if (hiveConf == null) { - System.exit(rc); + System.exit(RC); } } @@ -747,7 +750,13 @@ public class HiveStrictManagedMigration { return true; } - Path tablePath = new Path(tableObj.getSd().getLocation()); + String tablePathString = tableObj.getSd().getLocation(); + if (StringUtils.isEmpty(tablePathString)) { + // When using this tool in full automatic mode (no DB/table regexes and automatic migration option) we may + // encounter sysdb / information_schema databases. These should not be moved, they have null location. + return true; + } + Path tablePath = new Path(tablePathString); boolean shouldMoveTable = modifyLocation && ( (MANAGED_TABLE.name().equals(tableObj.getTableType()) && runOptions.shouldModifyManagedTableLocation) || diff --git a/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java b/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java index 16d4772..038a8a2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.hive.ql.TxnCommandsBaseForTests.Table.NONACIDORC import static org.apache.hadoop.hive.ql.TxnCommandsBaseForTests.Table.NONACIDORCTBL2; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; @@ -34,13 +35,17 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.ql.TxnCommandsBaseForTests; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import com.google.common.collect.ImmutableMap; + import org.junit.Assert; import org.junit.Test; @@ -65,6 +70,13 @@ public class TestHiveStrictManagedMigration extends TxnCommandsBaseForTests { runStatementOnDriver( "CREATE EXTERNAL TABLE texternal (a int, b int)"); + // Case for table having null location + runStatementOnDriver("CREATE EXTERNAL TABLE test.sysdbtest(tbl_id bigint)"); + org.apache.hadoop.hive.ql.metadata.Table table = Hive.get(hiveConf).getTable("test", "sysdbtest"); + table.getSd().unsetLocation(); + Hive.get(hiveConf).alterTable(table, false, + new EnvironmentContext(ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE)), false); + String oldWarehouse = getWarehouseDir(); String[] args = {"--hiveconf", "hive.strict.managed.tables=true", "-m", "automatic", "--modifyManagedTables", "--oldWarehouseRoot", oldWarehouse}; @@ -138,6 +150,9 @@ public class TestHiveStrictManagedMigration extends TxnCommandsBaseForTests { HiveStrictManagedMigration.hiveConf = hiveConf; HiveStrictManagedMigration.scheme = "file"; HiveStrictManagedMigration.main(args); + if (HiveStrictManagedMigration.RC != 0) { + fail("HiveStrictManagedMigration failed with error(s)"); + } } private void setupExternalTableTest() throws Exception {