This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e446776219 Update delete rows to be idempotent by always keeping at
least 1 tablet (#4008)
e446776219 is described below
commit e446776219e1bb2ff90d3e8f8f1f89128eb812b0
Author: Christopher L. Shannon <christopher.l.shan...@gmail.com>
AuthorDate: Sat Dec 2 09:36:07 2023 -0500
Update delete rows to be idempotent by always keeping at least 1 tablet
(#4008)
This fixes an edge case bug where delete rows could fail to recover if
all tablets are removed during delete and the operation fails before a
new default tablet can be created which will prevent the fate operatino
from resuming. Instead, now if the last tablet is included in the delete
range (null end row) we just keep the tablet and only remove the files
and update that tablets prev end row instead of creating a new tablet
which will guarantee we always have at least 1 tablet remaining and we
can recover the operation if it fails at any point.
---
.../accumulo/manager/TabletGroupWatcher.java | 71 ++++++++++++++--------
1 file changed, 47 insertions(+), 24 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 4ffeee967d..3295cb9604 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -98,11 +97,11 @@ import org.apache.accumulo.server.manager.state.MergeState;
import org.apache.accumulo.server.manager.state.TabletStateStore;
import org.apache.accumulo.server.manager.state.UnassignedTablet;
import org.apache.accumulo.server.tablets.TabletTime;
-import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterators;
@@ -732,30 +731,38 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
ample.putGcFileAndDirCandidates(extent.tableId(), datafilesAndDirs);
BatchWriter bw = client.createBatchWriter(targetSystemTable);
+ KeyExtent lastTabletInRange;
try {
- deleteTablets(info, deleteRange, bw, client);
+ lastTabletInRange = deleteTablets(info, deleteRange, bw, client);
} finally {
bw.close();
}
+ // If there is another tablet after the delete range then update the
prev end row
+ // of that tablet as all tablets will have been deleted in the delete
range.
+ // If the delete range includes the last tablet in the tablet then we
need
+ // to update the last tablet's previous end row as deleteTablets() will
keep the
+ // last tablet and only delete the files as we require at least 1 tablet
to exist
+ final KeyExtent goalTablet;
if (followingTablet != null) {
- Manager.log.debug("Updating prevRow of {} to {}", followingTablet,
extent.prevEndRow());
- bw = client.createBatchWriter(targetSystemTable);
- try {
- Mutation m = new Mutation(followingTablet.toMetaRow());
- TabletColumnFamily.PREV_ROW_COLUMN.put(m,
- TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
- ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
- bw.addMutation(m);
- bw.flush();
- } finally {
- bw.close();
- }
+ goalTablet = followingTablet;
} else {
- // Recreate the default tablet to hold the end of the table
- MetadataTableUtil.addTablet(new KeyExtent(extent.tableId(), null,
extent.prevEndRow()),
- ServerColumnFamily.DEFAULT_TABLET_DIR_NAME, manager.getContext(),
- metadataTime.getType(), manager.managerLock);
+ goalTablet = lastTabletInRange;
+ Preconditions.checkState(goalTablet != null && goalTablet.endRow() ==
null,
+ "If followingTablet is null then last tablet in delete range
should be the last tablet in the table");
+ }
+
+ Manager.log.debug("Updating prevRow of {} to {}", goalTablet,
extent.prevEndRow());
+ bw = client.createBatchWriter(targetSystemTable);
+ try {
+ Mutation m = new Mutation(goalTablet.toMetaRow());
+ TabletColumnFamily.PREV_ROW_COLUMN.put(m,
+ TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
+ ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+ bw.addMutation(m);
+ bw.flush();
+ } finally {
+ bw.close();
}
} catch (RuntimeException | TableNotFoundException ex) {
throw new AccumuloException(ex);
@@ -880,8 +887,8 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
}
- private void deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw,
AccumuloClient client)
- throws TableNotFoundException, MutationsRejectedException {
+ private KeyExtent deleteTablets(MergeInfo info, Range scanRange, BatchWriter
bw,
+ AccumuloClient client) throws TableNotFoundException, AccumuloException {
Scanner scanner;
Mutation m;
// Delete everything in the other tablets
@@ -893,24 +900,40 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
Manager.log.debug("Deleting range {}", scanRange);
scanner.setRange(scanRange);
RowIterator rowIter = new RowIterator(scanner);
+ KeyExtent lastTablet = null;
while (rowIter.hasNext()) {
Iterator<Entry<Key,Value>> row = rowIter.next();
m = null;
while (row.hasNext()) {
Entry<Key,Value> entry = row.next();
Key key = entry.getKey();
+ lastTablet = KeyExtent.fromMetaRow(key.getRow());
if (m == null) {
m = new Mutation(key.getRow());
}
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
- Manager.log.debug("deleting entry {}", key);
+ // For delete operations, check if this is the last tablet
+ // If this is the last tablet in the table then only delete
+ // the files as we need to make sure we always keep at least 1 tablet
+ if (info.isDelete() && lastTablet.endRow() == null) {
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ Manager.log.debug("deleting file entry {}", key);
+ }
+ } else {
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ Manager.log.debug("deleting entry {}", key);
+ }
+ }
+ if (m != null && !m.getUpdates().isEmpty()) {
+ bw.addMutation(m);
}
- bw.addMutation(m);
}
bw.flush();
+
+ return lastTablet;
}
private boolean isTabletAssigned(Key key) {
