This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push: new 09e3d77 Handle old sorted map files in Upgrade (#2185) 09e3d77 is described below commit 09e3d771ef11fcc2c2a8eb1d1b29b024d7da900f Author: Mike Miller <mmil...@apache.org> AuthorDate: Fri Jul 30 12:43:30 2021 -0400 Handle old sorted map files in Upgrade (#2185) * Add dropSortedMapWALFiles to Upgrader9to10 for resolving sorted map files that may still be around during upgrade. Follow up for #2117 * Closes #2179 --- .../accumulo/manager/upgrade/Upgrader9to10.java | 41 +++++++++++++++++++++ .../manager/upgrade/Upgrader9to10Test.java | 42 ++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java index 5e874c5..597f88c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java @@ -105,6 +105,12 @@ import com.google.common.base.Preconditions; * <a href="https://github.com/apache/accumulo/issues/1642">#1642</a>, and * <a href="https://github.com/apache/accumulo/issues/1643">#1643</a> as well.</li> * </ul> + * + * Sorted recovery was updated to use RFiles instead of map files. So to prevent issues during + * tablet recovery, remove the old temporary map files and resort using RFiles. This is done in + * {@link #dropSortedMapWALFiles(VolumeManager)}. For more information see the following issues: + * <a href="https://github.com/apache/accumulo/issues/2117">#2117</a> and + * <a href="https://github.com/apache/accumulo/issues/2179">#2179</a> */ public class Upgrader9to10 implements Upgrader { @@ -142,6 +148,8 @@ public class Upgrader9to10 implements Upgrader { upgradeRelativePaths(ctx, Ample.DataLevel.USER); upgradeDirColumns(ctx, Ample.DataLevel.USER); upgradeFileDeletes(ctx, Ample.DataLevel.USER); + // special case where old files need to be deleted + dropSortedMapWALFiles(ctx.getVolumeManager()); } private void setMetaTableProps(ServerContext ctx) { @@ -726,4 +734,37 @@ public class Upgrader9to10 implements Upgrader { } return new Path(upgradeProperty, VolumeManager.FileType.TABLE.getDirectory() + oldDelete); } + + /** + * Remove old temporary map files to prevent problems during recovery. + */ + static void dropSortedMapWALFiles(VolumeManager vm) { + Path recoveryDir = new Path("/accumulo/recovery"); + try { + if (!vm.exists(recoveryDir)) { + log.info("There are no recovery files in /accumulo/recovery"); + return; + } + List<Path> directoriesToDrop = new ArrayList<>(); + for (FileStatus walDir : vm.listStatus(recoveryDir)) { + // map files will be in a directory starting with "part" + Path walDirPath = walDir.getPath(); + for (FileStatus dirOrFile : vm.listStatus(walDirPath)) { + if (dirOrFile.isDirectory()) { + directoriesToDrop.add(walDirPath); + break; + } + } + } + if (!directoriesToDrop.isEmpty()) { + log.info("Found {} old sorted map directories to delete.", directoriesToDrop.size()); + for (Path dir : directoriesToDrop) { + log.info("Deleting everything in old sorted map directory: {}", dir); + vm.deleteRecursively(dir); + } + } + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java index a5752b9..25800da 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java @@ -24,11 +24,13 @@ import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -49,6 +51,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.gc.GcVolumeUtil; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.junit.Test; @@ -324,4 +327,43 @@ public class Upgrader9to10Test { assertEquals("Replacements should have update for every delete", deleteCount, updateCount); } + + @Test + public void testDropSortedMapWALs() throws IOException { + Path recoveryDir = new Path("/accumulo/recovery"); + VolumeManager fs = createMock(VolumeManager.class); + FileStatus[] dirs = new FileStatus[2]; + dirs[0] = createMock(FileStatus.class); + Path dir0 = new Path("/accumulo/recovery/A123456789"); + FileStatus[] dir0Files = new FileStatus[1]; + dir0Files[0] = createMock(FileStatus.class); + dirs[1] = createMock(FileStatus.class); + Path dir1 = new Path("/accumulo/recovery/B123456789"); + FileStatus[] dir1Files = new FileStatus[1]; + dir1Files[0] = createMock(FileStatus.class); + Path part1Dir = new Path("/accumulo/recovery/B123456789/part-r-0000"); + + expect(fs.exists(recoveryDir)).andReturn(true).once(); + expect(fs.listStatus(recoveryDir)).andReturn(dirs).once(); + expect(dirs[0].getPath()).andReturn(dir0).once(); + expect(fs.listStatus(dir0)).andReturn(dir0Files).once(); + expect(dir0Files[0].isDirectory()).andReturn(false).once(); + + expect(dirs[1].getPath()).andReturn(dir1).once(); + expect(fs.listStatus(dir1)).andReturn(dir1Files).once(); + expect(dir1Files[0].isDirectory()).andReturn(true).once(); + expect(dir1Files[0].getPath()).andReturn(part1Dir).once(); + + expect(fs.deleteRecursively(dir1)).andReturn(true).once(); + + replay(fs, dirs[0], dirs[1], dir0Files[0], dir1Files[0]); + Upgrader9to10.dropSortedMapWALFiles(fs); + + reset(fs); + + // test case where there is no recovery + expect(fs.exists(recoveryDir)).andReturn(false).once(); + replay(fs); + Upgrader9to10.dropSortedMapWALFiles(fs); + } }