This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push: new e446776219 Update delete rows to be idempotent by always keeping at least 1 tablet (#4008) e446776219 is described below commit e446776219e1bb2ff90d3e8f8f1f89128eb812b0 Author: Christopher L. Shannon <christopher.l.shan...@gmail.com> AuthorDate: Sat Dec 2 09:36:07 2023 -0500 Update delete rows to be idempotent by always keeping at least 1 tablet (#4008) This fixes an edge case bug where delete rows could fail to recover if all tablets are removed during delete and the operation fails before a new default tablet can be created which will prevent the fate operatino from resuming. Instead, now if the last tablet is included in the delete range (null end row) we just keep the tablet and only remove the files and update that tablets prev end row instead of creating a new tablet which will guarantee we always have at least 1 tablet remaining and we can recover the operation if it fails at any point. --- .../accumulo/manager/TabletGroupWatcher.java | 71 ++++++++++++++-------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 4ffeee967d..3295cb9604 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.RowIterator; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; @@ -98,11 +97,11 @@ import org.apache.accumulo.server.manager.state.MergeState; import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.server.manager.state.UnassignedTablet; import org.apache.accumulo.server.tablets.TabletTime; -import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.thrift.TException; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterators; @@ -732,30 +731,38 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { } ample.putGcFileAndDirCandidates(extent.tableId(), datafilesAndDirs); BatchWriter bw = client.createBatchWriter(targetSystemTable); + KeyExtent lastTabletInRange; try { - deleteTablets(info, deleteRange, bw, client); + lastTabletInRange = deleteTablets(info, deleteRange, bw, client); } finally { bw.close(); } + // If there is another tablet after the delete range then update the prev end row + // of that tablet as all tablets will have been deleted in the delete range. + // If the delete range includes the last tablet in the tablet then we need + // to update the last tablet's previous end row as deleteTablets() will keep the + // last tablet and only delete the files as we require at least 1 tablet to exist + final KeyExtent goalTablet; if (followingTablet != null) { - Manager.log.debug("Updating prevRow of {} to {}", followingTablet, extent.prevEndRow()); - bw = client.createBatchWriter(targetSystemTable); - try { - Mutation m = new Mutation(followingTablet.toMetaRow()); - TabletColumnFamily.PREV_ROW_COLUMN.put(m, - TabletColumnFamily.encodePrevEndRow(extent.prevEndRow())); - ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m); - bw.addMutation(m); - bw.flush(); - } finally { - bw.close(); - } + goalTablet = followingTablet; } else { - // Recreate the default tablet to hold the end of the table - MetadataTableUtil.addTablet(new KeyExtent(extent.tableId(), null, extent.prevEndRow()), - ServerColumnFamily.DEFAULT_TABLET_DIR_NAME, manager.getContext(), - metadataTime.getType(), manager.managerLock); + goalTablet = lastTabletInRange; + Preconditions.checkState(goalTablet != null && goalTablet.endRow() == null, + "If followingTablet is null then last tablet in delete range should be the last tablet in the table"); + } + + Manager.log.debug("Updating prevRow of {} to {}", goalTablet, extent.prevEndRow()); + bw = client.createBatchWriter(targetSystemTable); + try { + Mutation m = new Mutation(goalTablet.toMetaRow()); + TabletColumnFamily.PREV_ROW_COLUMN.put(m, + TabletColumnFamily.encodePrevEndRow(extent.prevEndRow())); + ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m); + bw.addMutation(m); + bw.flush(); + } finally { + bw.close(); } } catch (RuntimeException | TableNotFoundException ex) { throw new AccumuloException(ex); @@ -880,8 +887,8 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { } } - private void deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw, AccumuloClient client) - throws TableNotFoundException, MutationsRejectedException { + private KeyExtent deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw, + AccumuloClient client) throws TableNotFoundException, AccumuloException { Scanner scanner; Mutation m; // Delete everything in the other tablets @@ -893,24 +900,40 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { Manager.log.debug("Deleting range {}", scanRange); scanner.setRange(scanRange); RowIterator rowIter = new RowIterator(scanner); + KeyExtent lastTablet = null; while (rowIter.hasNext()) { Iterator<Entry<Key,Value>> row = rowIter.next(); m = null; while (row.hasNext()) { Entry<Key,Value> entry = row.next(); Key key = entry.getKey(); + lastTablet = KeyExtent.fromMetaRow(key.getRow()); if (m == null) { m = new Mutation(key.getRow()); } - m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); - Manager.log.debug("deleting entry {}", key); + // For delete operations, check if this is the last tablet + // If this is the last tablet in the table then only delete + // the files as we need to make sure we always keep at least 1 tablet + if (info.isDelete() && lastTablet.endRow() == null) { + if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { + m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); + Manager.log.debug("deleting file entry {}", key); + } + } else { + m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); + Manager.log.debug("deleting entry {}", key); + } + } + if (m != null && !m.getUpdates().isEmpty()) { + bw.addMutation(m); } - bw.addMutation(m); } bw.flush(); + + return lastTablet; } private boolean isTabletAssigned(Key key) {