arp7 commented on a change in pull request #1129:
URL: https://github.com/apache/hadoop-ozone/pull/1129#discussion_r455442847
##########
File path:
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
##########
@@ -3041,58 +3047,74 @@ public TermIndex installSnapshot(String leaderId) {
omRatisServer.getOmStateMachine().pause();
} catch (Exception e) {
LOG.error("Failed to stop/ pause the services. Cannot proceed with " +
- "installing the new checkpoint.", e);
- return null;
- }
-
- //TODO: un-pause SM if any failures and retry?
-
- long lastAppliedIndex = omRatisServer.getLastAppliedTermIndex().getIndex();
-
- boolean canProceed =
- OzoneManagerRatisUtils.verifyTransactionInfo(omTransactionInfo,
- lastAppliedIndex, leaderId, newDBlocation);
-
- // If downloaded DB has transaction info less than current one, return.
- if (!canProceed) {
- return null;
+ "installing the new checkpoint.");
+ // During stopServices, if KeyManager was stopped successfully and
+ // OMMetadataManager stop failed, we should restart the KeyManager.
+ keyManager.start(configuration);
+ throw e;
}
- long leaderIndex = omTransactionInfo.getTransactionIndex();
- long leaderTerm = omTransactionInfo.getCurrentTerm();
+ File dbBackup = null;
+ TermIndex termIndex = omRatisServer.getLastAppliedTermIndex();
+ long term = termIndex.getTerm();
+ long lastAppliedIndex = termIndex.getIndex();
+ // Check if current applied log index is smaller than the downloaded
+ // checkpoint transaction index. If yes, proceed by stopping the ratis
+ // server so that the OM state can be re-initialized. If no then do not
+ // proceed with installSnapshot.
+ boolean canProceed = OzoneManagerRatisUtils.verifyTransactionInfo(
+ checkpointTrxnInfo, lastAppliedIndex, leaderId, checkpointLocation);
- File dbBackup;
- try {
- dbBackup = replaceOMDBWithCheckpoint(lastAppliedIndex, oldDBLocation,
- newDBlocation);
- } catch (Exception e) {
- LOG.error("OM DB checkpoint replacement with new downloaded checkpoint "
+
- "failed.", e);
- return null;
+ if (canProceed) {
+ try {
+ dbBackup = replaceOMDBWithCheckpoint(lastAppliedIndex, oldDBLocation,
+ checkpointLocation);
+ term = checkpointTrxnInfo.getTerm();
+ lastAppliedIndex = checkpointTrxnInfo.getTransactionIndex();
+ LOG.info("Replaced DB with checkpoint from OM: {}, term: {}, index:
{}",
+ leaderId, term, lastAppliedIndex);
+ } catch (Exception e) {
+ LOG.error("Failed to install Snapshot from {} as OM failed to replace"
+
+ " DB with downloaded checkpoint. Reloading old OM state.", e);
+ }
+ } else {
+ LOG.warn("Cannot proceed with InstallSnapshot as OM is at TermIndex {} "
+
+ "and checkpoint has lower TermIndex {}. Reloading old state of OM.",
+ termIndex, checkpointTrxnInfo.getTermIndex());
}
// Reload the OM DB store with the new checkpoint.
// Restart (unpause) the state machine and update its last applied index
// to the installed checkpoint's snapshot index.
try {
- reloadOMState(leaderIndex, leaderTerm);
- omRatisServer.getOmStateMachine().unpause(leaderIndex, leaderTerm);
- } catch (IOException e) {
- LOG.error("Failed to reload OM state with new DB checkpoint.", e);
- return null;
+ reloadOMState(lastAppliedIndex, term);
+ omRatisServer.getOmStateMachine().unpause(lastAppliedIndex, term);
+ LOG.info("Reloaded OM state with Term: {} and Index: {}", term,
+ lastAppliedIndex);
+ } catch (IOException ex) {
+ String errorMsg = "Failed to reload OM state and instantiate services.";
+ exitManager.exitSystem(1, errorMsg, ex, LOG);
}
// Delete the backup DB
try {
- FileUtils.deleteFully(dbBackup);
+ if (dbBackup != null) {
+ FileUtils.deleteFully(dbBackup);
+ }
Review comment:
On successful restart we should clear any old backups.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: ozone-issues-h...@hadoop.apache.org
