Repository: hive Updated Branches: refs/heads/master 8d084d676 -> 881e29124
HIVE-20998: HiveStrictManagedMigration utility should update DB/Table location as last migration steps (Jason Dere, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/881e2912 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/881e2912 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/881e2912 Branch: refs/heads/master Commit: 881e291246a367f0a413da3bb13be410923a37cf Parents: 8d084d6 Author: Jason Dere <jd...@hortonworks.com> Authored: Wed Dec 12 14:33:27 2018 -0800 Committer: Jason Dere <jd...@hortonworks.com> Committed: Wed Dec 12 14:33:27 2018 -0800 ---------------------------------------------------------------------- .../ql/util/HiveStrictManagedMigration.java | 38 +++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/881e2912/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index 9535bed..80025b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -404,11 +404,6 @@ public class HiveStrictManagedMigration { // Set appropriate owner/perms of the DB dir only, no need to recurse checkAndSetFileOwnerPermissions(fs, newDefaultDbLocation, ownerName, groupName, dirPerms, null, runOptions.dryRun, false); - - // The table processing needs the db location at the old location, so clone the DB object - // when updating the location. - Database modifiedDb = dbObj.deepCopy(); - getHiveUpdater().updateDbLocation(modifiedDb, newDefaultDbLocation); } } @@ -416,6 +411,7 @@ public class HiveStrictManagedMigration { createExternalDbDir(dbObj); } + boolean errorsInThisDb = false; List<String> tableNames = hms.getTables(dbName, runOptions.tableRegex); for (String tableName : tableNames) { // If we did not change the DB location, there is no need to move the table directories. @@ -424,6 +420,20 @@ public class HiveStrictManagedMigration { } catch (Exception err) { LOG.error("Error processing table " + getQualifiedName(dbObj.getName(), tableName), err); failuresEncountered = true; + errorsInThisDb = true; + } + } + + // Finally update the DB location. This would prevent subsequent runs of the migration from processing this DB. + if (modifyDefaultManagedLocation) { + if (errorsInThisDb) { + LOG.error("Not updating database location for {} since an error was encountered. The migration must be run again for this database.", + dbObj.getName()); + } else { + Path newDefaultDbLocation = wh.getDefaultDatabasePath(dbName); + // dbObj after this call would have the new DB location. + // Keep that in mind if anything below this requires the old DB path. + getHiveUpdater().updateDbLocation(dbObj, newDefaultDbLocation); } } } @@ -590,6 +600,8 @@ public class HiveStrictManagedMigration { Path oldTablePath = new Path(tableObj.getSd().getLocation()); LOG.info("Moving location of {} from {} to {}", getQualifiedName(tableObj), oldTablePath, newTablePath); + + // Move table directory. if (!runOptions.dryRun) { FileSystem fs = newTablePath.getFileSystem(conf); if (fs.exists(oldTablePath)) { @@ -601,9 +613,13 @@ public class HiveStrictManagedMigration { } } } - if (!runOptions.dryRun) { - getHiveUpdater().updateTableLocation(tableObj, newTablePath); - } + + // An error occurring between here and before updating the table's location in the metastore + // may potentially cause the data to reside in the new location, while the + // table/partitions point to the old paths. + // The migration would be _REQUIRED_ to run again (and pass) for the data and table/partition + // locations to be in sync. + if (isPartitionedTable(tableObj)) { List<String> partNames = hms.listPartitionNames(dbName, tableName, Short.MAX_VALUE); // TODO: Fetch partitions in batches? @@ -622,6 +638,12 @@ public class HiveStrictManagedMigration { } } } + + // Finally update the table location. This would prevent this tool from processing this table again + // on subsequent runs of the migration. + if (!runOptions.dryRun) { + getHiveUpdater().updateTableLocation(tableObj, newTablePath); + } } static void renameFilesToConformToAcid(Table tableObj, IMetaStoreClient hms, Configuration conf, boolean dryRun)
