Repository: hive Updated Branches: refs/heads/branch-3 5773bca1f -> 2b57dd27a
HIVE-19880: Repl Load to return recoverable vs non-recoverable error codes (Mahesh Kumar Behera, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2b57dd27 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2b57dd27 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2b57dd27 Branch: refs/heads/branch-3 Commit: 2b57dd27ad61e552f93817ac69313066af6562d9 Parents: 5773bca Author: Sankar Hariappan <sank...@apache.org> Authored: Wed Jun 20 13:54:25 2018 -0700 Committer: Sankar Hariappan <sank...@apache.org> Committed: Wed Jun 20 13:54:25 2018 -0700 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 31 ++++++++++++++++++-- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 22 ++++++++++++++ .../org/apache/hadoop/hive/ql/ErrorMsg.java | 11 +++++++ .../hadoop/hive/ql/exec/ReplCopyTask.java | 3 +- .../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 3 +- .../ql/exec/repl/bootstrap/ReplLoadTask.java | 3 +- .../filesystem/DatabaseEventsIterator.java | 4 +-- .../ql/parse/ReplicationSemanticAnalyzer.java | 20 ++++++------- .../hadoop/hive/ql/parse/repl/CopyUtils.java | 16 +++++----- .../hive/ql/parse/repl/dump/TableExport.java | 2 +- .../ql/parse/repl/dump/io/FileOperations.java | 5 ++-- .../hive/metastore/HiveMetaStoreClient.java | 5 +++- .../hive/metastore/messaging/EventUtils.java | 4 +-- 13 files changed, 98 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 862140f..d161841 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -73,9 +73,9 @@ import org.junit.rules.TestName; import org.junit.rules.TestRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import javax.annotation.Nullable; - import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -853,7 +853,8 @@ public class TestReplicationScenarios { InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventIdSkipper); advanceDumpDir(); - verifyFail("REPL DUMP " + dbName + " FROM " + replDumpId, driver); + CommandProcessorResponse ret = driver.run("REPL DUMP " + dbName + " FROM " + replDumpId); + assertTrue(ret.getResponseCode() == ErrorMsg.REPL_EVENTS_MISSING_IN_METASTORE.getErrorCode()); eventIdSkipper.assertInjectionsPerformed(true,false); InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour } @@ -3158,6 +3159,32 @@ public class TestReplicationScenarios { } @Test + public void testLoadCmPathMissing() throws IOException { + String dbName = createDB(testName.getMethodName(), driver); + run("CREATE TABLE " + dbName + ".normal(a int)", driver); + run("INSERT INTO " + dbName + ".normal values (1)", driver); + + advanceDumpDir(); + run("repl dump " + dbName, true, driver); + String dumpLocation = getResult(0, 0, driver); + + run("DROP TABLE " + dbName + ".normal", driver); + + String cmDir = hconf.getVar(HiveConf.ConfVars.REPLCMDIR); + Path path = new Path(cmDir); + FileSystem fs = path.getFileSystem(hconf); + ContentSummary cs = fs.getContentSummary(path); + long fileCount = cs.getFileCount(); + assertTrue(fileCount != 0); + fs.delete(path); + + CommandProcessorResponse ret = driverMirror.run("REPL LOAD " + dbName + " FROM '" + dumpLocation + "'"); + assertTrue(ret.getResponseCode() == ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getErrorCode()); + run("drop database " + dbName, true, driver); + fs.create(path, false); + } + + @Test public void testDumpNonReplDatabase() throws IOException { String dbName = createDBNonRepl(testName.getMethodName(), driver); verifyFail("REPL DUMP " + dbName, driver); http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index b217259..f597bef 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -45,6 +45,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -2927,6 +2928,27 @@ public class TestJdbcDriver2 { stmt.close(); } + @Test + public void testReplErrorScenarios() throws Exception { + HiveStatement stmt = (HiveStatement) con.createStatement(); + + try { + // source of replication not set + stmt.execute("repl dump default"); + } catch(SQLException e){ + assertTrue(e.getErrorCode() == ErrorMsg.REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION.getErrorCode()); + } + + try { + // invalid load path + stmt.execute("repl load default1 from '/tmp/junk'"); + } catch(SQLException e){ + assertTrue(e.getErrorCode() == ErrorMsg.REPL_LOAD_PATH_NOT_FOUND.getErrorCode()); + } + + stmt.close(); + } + /** * Test {@link HiveStatement#executeAsync(String)} for an insert overwrite into a table * @throws Exception http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 63ce36a..693ab25 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -499,6 +499,15 @@ public enum ErrorMsg { + " Please fix and try again.", true), SPARK_RUNTIME_OOM(20015, "Spark job failed because of out of memory."), + //if the error message is changed for REPL_EVENTS_MISSING_IN_METASTORE, then need modification in getNextNotification + //method in HiveMetaStoreClient + REPL_EVENTS_MISSING_IN_METASTORE(20016, "Notification events are missing in the meta store."), + REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID(20017, "Target database is bootstrapped from some other path."), + REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH(20018, "File is missing from both source and cm path."), + REPL_LOAD_PATH_NOT_FOUND(20019, "Load path does not exist."), + REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION(20020, + "Source of replication (repl.source.for) is not set in the database properties."), + // An exception from runtime that will show the full stack to client UNRESOLVED_RT_EXCEPTION(29999, "Runtime Error: {0}", "58004", true), @@ -586,6 +595,8 @@ public enum ErrorMsg { SPARK_JOB_INTERRUPTED(30044, "Spark job was interrupted while executing"), + REPL_FILE_SYSTEM_OPERATION_RETRY(30045, "Replication file system operation retry expired."), + //========================== 40000 range starts here ========================// SPARK_JOB_RUNTIME_ERROR(40001, http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java index 5fecf81..4ace861 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.ReplCopyWork; import org.apache.hadoop.hive.ql.parse.repl.CopyUtils; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.BufferedReader; import java.io.IOException; @@ -165,7 +166,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable { } catch (Exception e) { console.printError("Failed with exception " + e.getMessage(), "\n" + StringUtils.stringifyException(e)); - return (1); + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index ccdf04a..78a0ba3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hive.ql.plan.ExportWork.MmContext; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.Serializable; import java.util.ArrayList; @@ -123,7 +124,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable { } catch (Exception e) { LOG.error("failed", e); setException(e); - return 1; + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } return 0; } http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java index 76fb2a3..b33a774 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.Serializable; import java.util.ArrayList; @@ -223,7 +224,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable { } catch (Exception e) { LOG.error("failed replication", e); setException(e); - return 1; + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } LOG.info("completed load task run : {}", work.executedLoadTask()); return 0; http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java index ecedf9b..f778cb4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java @@ -91,8 +91,8 @@ class DatabaseEventsIterator implements Iterator<BootstrapEvent> { return true; } catch (Exception e) { // may be do some retry logic here. - throw new RuntimeException("could not traverse the file via remote iterator " + dbLevelPath, - e); + LOG.error("could not traverse the file via remote iterator " + dbLevelPath, e); + throw new RuntimeException(e.getMessage(), e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index c3bb132..f37de3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -61,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG; @@ -110,7 +111,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { try { initReplDump(ast); } catch (HiveException e) { - throw new SemanticException("repl dump failed " + e.getMessage()); + throw new SemanticException(e.getMessage(), e); } analyzeReplDump(ast); break; @@ -147,11 +148,8 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { if (null != replConfigs) { for (Map.Entry<String, String> config : replConfigs.entrySet()) { conf.set(config.getKey(), config.getValue()); - if ("hive.repl.dump.metadata.only".equalsIgnoreCase(config.getKey()) && - "true".equalsIgnoreCase(config.getValue())) { - isMetaDataOnly = true; - } } + isMetaDataOnly = HiveConf.getBoolVar(conf, REPL_DUMP_METADATA_ONLY); } } else if (ast.getChild(currNode).getType() == TOK_TABNAME) { // optional tblName was specified. @@ -185,9 +183,10 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { Database database = db.getDatabase(dbName); if (database != null) { - if (!ReplChangeManager.isSourceOfReplication(database) && !isMetaDataOnly) { - throw new SemanticException("Cannot dump database " + dbName + - " as it is not a source of replication"); + if (!isMetaDataOnly && !ReplChangeManager.isSourceOfReplication(database)) { + LOG.error("Cannot dump database " + dbName + + " as it is not a source of replication (repl.source.for)"); + throw new SemanticException(ErrorMsg.REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION.getMsg()); } } else { throw new SemanticException("Cannot dump database " + dbName + " as it does not exist"); @@ -366,7 +365,8 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { if (!fs.exists(loadPath)) { // supposed dump path does not exist. - throw new FileNotFoundException(loadPath.toUri().toString()); + LOG.error("File not found " + loadPath.toUri().toString()); + throw new FileNotFoundException(ErrorMsg.REPL_LOAD_PATH_NOT_FOUND.getMsg()); } // Now, the dumped path can be one of three things: @@ -512,7 +512,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { } catch (Exception e) { // TODO : simple wrap & rethrow for now, clean up with error codes - throw new SemanticException(e); + throw new SemanticException(e.getMessage(), e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java index 79b4652..7e8d520 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java @@ -31,6 +31,8 @@ import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.metadata.HiveFatalException; import javax.security.auth.login.LoginException; import java.io.FileNotFoundException; @@ -68,7 +70,7 @@ public class CopyUtils { // changed/removed during copy, so double check the checksum after copy, // if not match, copy again from cm public void copyAndVerify(FileSystem destinationFs, Path destRoot, - List<ReplChangeManager.FileInfo> srcFiles) throws IOException, LoginException { + List<ReplChangeManager.FileInfo> srcFiles) throws IOException, LoginException, HiveFatalException { Map<FileSystem, Map< Path, List<ReplChangeManager.FileInfo>>> map = fsToFileMap(srcFiles, destRoot); for (Map.Entry<FileSystem, Map<Path, List<ReplChangeManager.FileInfo>>> entry : map.entrySet()) { FileSystem sourceFs = entry.getKey(); @@ -92,7 +94,7 @@ public class CopyUtils { private void doCopyRetry(FileSystem sourceFs, List<ReplChangeManager.FileInfo> srcFileList, FileSystem destinationFs, Path destination, - boolean useRegularCopy) throws IOException, LoginException { + boolean useRegularCopy) throws IOException, LoginException, HiveFatalException { int repeat = 0; boolean isCopyError = false; List<Path> pathList = Lists.transform(srcFileList, ReplChangeManager.FileInfo::getEffectivePath); @@ -145,7 +147,7 @@ public class CopyUtils { // If still files remains to be copied due to failure/checksum mismatch after several attempts, then throw error if (!pathList.isEmpty()) { LOG.error("File copy failed even after several attempts. Files list: " + pathList); - throw new IOException("File copy failed even after several attempts."); + throw new IOException(ErrorMsg.REPL_FILE_SYSTEM_OPERATION_RETRY.getMsg()); } } @@ -154,7 +156,7 @@ public class CopyUtils { // itself is missing, then throw error. private List<Path> getFilesToRetry(FileSystem sourceFs, List<ReplChangeManager.FileInfo> srcFileList, FileSystem destinationFs, Path destination, boolean isCopyError) - throws IOException { + throws IOException, HiveFatalException { List<Path> pathList = new ArrayList<Path>(); // Going through file list and make the retry list @@ -190,9 +192,9 @@ public class CopyUtils { srcPath = srcFile.getEffectivePath(); if (null == srcPath) { // This case possible if CM path is not enabled. - LOG.error("File copy failed and likely source file is deleted or modified. " + LOG.error("File copy failed and likely source file is deleted or modified." + "Source File: " + srcFile.getSourcePath()); - throw new IOException("File copy failed and likely source file is deleted or modified."); + throw new HiveFatalException(ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getMsg()); } if (!srcFile.isUseSourcePath() && !sourceFs.exists(srcFile.getCmPath())) { @@ -201,7 +203,7 @@ public class CopyUtils { + "Missing Source File: " + srcFile.getSourcePath() + ", CM File: " + srcFile.getCmPath() + ". " + "Try setting higher value for hive.repl.cm.retain in source warehouse. " + "Also, bootstrap the system again to get back the consistent replicated state."); - throw new IOException("Both source and CM path are missing from source."); + throw new HiveFatalException(ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getMsg()); } pathList.add(srcPath); http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java index 20ff23a..b60be88 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java @@ -162,7 +162,7 @@ public class TableExport { .export(replicationSpec); } } catch (Exception e) { - throw new SemanticException(e); + throw new SemanticException(e.getMessage(), e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 5e88927..070c830 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -21,12 +21,10 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import javax.security.auth.login.LoginException; -import org.apache.curator.shaded.com.google.common.collect.Lists; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,6 +32,7 @@ import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.ReplChangeManager; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -163,7 +162,7 @@ public class FileOperations { logger.info("writeFilesList failed", e); if (repeat >= FileUtils.MAX_IO_ERROR_RETRY) { logger.error("exporting data files in dir : " + dataPathList + " to " + exportRootDataDir + " failed"); - throw e; + throw new IOException(ErrorMsg.REPL_FILE_SYSTEM_OPERATION_RETRY.getMsg()); } int sleepTime = FileUtils.getSleepTime(repeat - 1); http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 27b3032..0eb7f1a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -130,6 +130,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { static final protected Logger LOG = LoggerFactory.getLogger(HiveMetaStoreClient.class); + //copied from ErrorMsg.java + private static final String REPL_EVENTS_MISSING_IN_METASTORE = "Notification events are missing in the meta store."; + public HiveMetaStoreClient(Configuration conf) throws MetaException { this(conf, null, true); } @@ -2706,7 +2709,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { + "Try setting higher value for hive.metastore.event.db.listener.timetolive. " + "Also, bootstrap the system again to get back the consistent replicated state.", nextEventId, e.getEventId()); - throw new IllegalStateException("Notification events are missing."); + throw new IllegalStateException(REPL_EVENTS_MISSING_IN_METASTORE); } if ((filter != null) && filter.accept(e)) { filtered.addToEvents(e); http://git-wip-us.apache.org/repos/asf/hive/blob/2b57dd27/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java index 7d8c1d4..2b16897 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java @@ -93,7 +93,7 @@ public class EventUtils { try { return msc.getNextNotification(pos,getBatchSize(), filter).getEvents(); } catch (TException e) { - throw new IOException(e); + throw new IOException(e.getMessage(), e); } } } @@ -179,7 +179,7 @@ public class EventUtils { // but throwing the exception is the appropriate result here, and hasNext() // signature will only allow RuntimeExceptions. Iterator.hasNext() really // should have allowed IOExceptions - throw new RuntimeException(e); + throw new RuntimeException(e.getMessage(), e); } // New batch has been fetched. If it's not empty, we have more elements to process. return !batch.isEmpty();
