[accumulo] branch main updated: Add retry counter for log recovery with MinC (#2084)

[jmanno]

This is an automated email from the ASF dual-hosted git repository.

jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 9d42e3b  Add retry counter for log recovery with MinC (#2084)
9d42e3b is described below

commit 9d42e3bf4cb3650b72b680f91855eeda6ff6ee53
Author: Jeffrey Manno <jeffreymann...@gmail.com>
AuthorDate: Wed Jun 16 13:46:41 2021 -0400

    Add retry counter for log recovery with MinC (#2084)
    
    * Add retrycounter for log recovery with MinC
---
 .../apache/accumulo/tserver/tablet/MinorCompactor.java | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
index 37dae21..c49f6d5 100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
@@ -62,6 +62,7 @@ public class MinorCompactor extends FileCompactor {
   private static final Logger log = 
LoggerFactory.getLogger(MinorCompactor.class);
 
   private final TabletServer tabletServer;
+  private final MinorCompactionReason mincReason;
 
   public MinorCompactor(TabletServer tabletServer, Tablet tablet, InMemoryMap 
imm,
       TabletFile outputFile, MinorCompactionReason mincReason, 
TableConfiguration tableConfig) {
@@ -113,6 +114,7 @@ public class MinorCompactor extends FileCompactor {
           }
         }, Collections.emptyList(), tableConfig);
     this.tabletServer = tabletServer;
+    this.mincReason = mincReason;
   }
 
   private boolean isTableDeleting() {
@@ -141,6 +143,7 @@ public class MinorCompactor extends FileCompactor {
     double growthFactor = 4;
     int maxSleepTime = 1000 * 60 * 3; // 3 minutes
     boolean reportedProblem = false;
+    int retryCounter = 0;
 
     runningCompactions.add(this);
     try {
@@ -165,12 +168,21 @@ public class MinorCompactor extends FileCompactor {
           reportedProblem = true;
         } catch (RuntimeException | NoClassDefFoundError e) {
           // if this is coming from a user iterator, it is possible that the 
user could change the
-          // iterator config and that the
-          // minor compaction would succeed
-          log.warn("MinC failed ({}) to create {} retrying ...", 
e.getMessage(), outputFileName, e);
+          // iterator config and that the minor compaction would succeed
+          // If the minor compaction stalls for too long during recovery, it 
can interfere with
+          // other tables loading
+          // Throw exception if this happens so assignments can be rescheduled.
           ProblemReports.getInstance(tabletServer.getContext()).report(
               new ProblemReport(getExtent().tableId(), ProblemType.FILE_WRITE, 
outputFileName, e));
+          if (retryCounter >= 4 && 
mincReason.equals(MinorCompactionReason.RECOVERY)) {
+            log.warn(
+                "MinC ({}) is stuck for too long during recovery, throwing 
error to reschedule.",
+                getExtent(), e);
+            throw new RuntimeException(e);
+          }
+          log.warn("MinC failed ({}) to create {} retrying ...", 
e.getMessage(), outputFileName, e);
           reportedProblem = true;
+          retryCounter++;
         } catch (CompactionCanceledException e) {
           throw new IllegalStateException(e);
         }