HBASE-20469 Directory used for sidelining old recovered edits files should be made configurable
Signed-off-by: Andrew Purtell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5520fa1a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5520fa1a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5520fa1a Branch: refs/heads/branch-1 Commit: 5520fa1ab373c2157d025dac46856602b5e58d34 Parents: 8510331 Author: Nihal Jain <[email protected]> Authored: Sat Apr 21 16:40:21 2018 +0530 Committer: Andrew Purtell <[email protected]> Committed: Wed Aug 15 18:01:43 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hbase/wal/WALSplitter.java | 10 ++++++---- .../java/org/apache/hadoop/hbase/wal/TestWALSplit.java | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/5520fa1a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 6ef1599..735549b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -541,6 +541,7 @@ public class WALSplitter { * creating it if necessary. * @param logEntry * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. + * @param tmpDirName of the directory used to sideline old recovered edits file * @param conf * @return Path to file into which to dump split log edits. * @throws IOException @@ -548,8 +549,7 @@ public class WALSplitter { @SuppressWarnings("deprecation") @VisibleForTesting static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit, - Configuration conf) - throws IOException { + String tmpDirName, Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); Path rootDir = FSUtils.getRootDir(conf); Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTablename()); @@ -564,7 +564,7 @@ public class WALSplitter { return null; } if (fs.exists(dir) && fs.isFile(dir)) { - Path tmp = new Path("/tmp"); + Path tmp = new Path(tmpDirName); if (!fs.exists(tmp)) { fs.mkdirs(tmp); } @@ -1589,8 +1589,10 @@ public class WALSplitter { * @return a path with a write for that path. caller should close. */ WriterAndPath createWAP(byte[] region, Entry entry) throws IOException { + String tmpDirName = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); Path regionedits = getRegionSplitEditsPath(entry, - fileBeingSplit.getPath().getName(), conf); + fileBeingSplit.getPath().getName(), tmpDirName, conf); if (regionedits == null) { return null; } http://git-wip-us.apache.org/repos/asf/hbase/blob/5520fa1a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index 04aee70..b6cfdc8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -123,6 +123,7 @@ public class TestWALSplit { private Path OLDLOGDIR; private Path CORRUPTDIR; private Path TABLEDIR; + private String TMPDIRNAME; private static final int NUM_WRITERS = 10; private static final int ENTRIES = 10; // entries per writer per region @@ -185,6 +186,8 @@ public class TestWALSplit { OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME); CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME); TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME); + TMPDIRNAME = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); REGIONS.clear(); Collections.addAll(REGIONS, "bbb", "ccc"); InstrumentedLogWriter.activateFailure = false; @@ -386,7 +389,7 @@ public class TestWALSplit { TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), new WALEdit()); Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, conf); + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); } @@ -412,7 +415,7 @@ public class TestWALSplit { fs.createNewFile(parent); // create a recovered.edits file Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, conf); + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close();
