HDFS-12912. [READ] Fix configuration and implementation of LevelDB-based alias maps
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80c3fec3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80c3fec3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80c3fec3 Branch: refs/heads/trunk Commit: 80c3fec3a13c41051daaae42e5c9a9fedf5c7ee7 Parents: c89b29b Author: Virajith Jalaparti <viraj...@apache.org> Authored: Wed Dec 13 13:39:21 2017 -0800 Committer: Chris Douglas <cdoug...@apache.org> Committed: Fri Dec 15 17:51:41 2017 -0800 ---------------------------------------------------------------------- .../hdfs/server/aliasmap/InMemoryAliasMap.java | 42 ++++++++++---------- .../aliasmap/InMemoryLevelDBAliasMapServer.java | 9 +++-- .../impl/LevelDBFileRegionAliasMap.java | 5 +++ .../src/site/markdown/HdfsProvidedStorage.md | 4 +- .../server/aliasmap/ITestInMemoryAliasMap.java | 9 +++-- .../server/aliasmap/TestInMemoryAliasMap.java | 2 +- .../impl/TestInMemoryLevelDBAliasMapClient.java | 2 + .../impl/TestLevelDbMockAliasMapClient.java | 2 +- .../TestNameNodeProvidedImplementation.java | 2 + 9 files changed, 45 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java index 3d9eeea..142a040 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java @@ -59,6 +59,7 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, private final DB levelDb; private Configuration conf; + private String blockPoolID; @Override public void setConf(Configuration conf) { @@ -79,32 +80,38 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, .toString(); } - public static @Nonnull InMemoryAliasMap init(Configuration conf) - throws IOException { + public static @Nonnull InMemoryAliasMap init(Configuration conf, + String blockPoolID) throws IOException { Options options = new Options(); options.createIfMissing(true); String directory = conf.get(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); LOG.info("Attempting to load InMemoryAliasMap from \"{}\"", directory); - File path = new File(directory); - if (!path.exists()) { + File levelDBpath; + if (blockPoolID != null) { + levelDBpath = new File(directory, blockPoolID); + } else { + levelDBpath = new File(directory); + } + if (!levelDBpath.exists()) { String error = createPathErrorMessage(directory); throw new IOException(error); } - DB levelDb = JniDBFactory.factory.open(path, options); - InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDb); + DB levelDb = JniDBFactory.factory.open(levelDBpath, options); + InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDb, blockPoolID); aliasMap.setConf(conf); return aliasMap; } @VisibleForTesting - InMemoryAliasMap(DB levelDb) { + InMemoryAliasMap(DB levelDb, String blockPoolID) { this.levelDb = levelDb; + this.blockPoolID = blockPoolID; } @Override public IterationResult list(Optional<Block> marker) throws IOException { - return withIterator((DBIterator iterator) -> { + try (DBIterator iterator = levelDb.iterator()) { Integer batchSize = conf.getInt(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_BATCH_SIZE, DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_BATCH_SIZE_DEFAULT); @@ -130,8 +137,7 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, } else { return new IterationResult(batch, Optional.empty()); } - - }); + } } public @Nonnull Optional<ProvidedStorageLocation> read(@Nonnull Block block) @@ -159,7 +165,7 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, @Override public String getBlockPoolId() { - return null; + return blockPoolID; } public void close() throws IOException { @@ -202,21 +208,15 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, return blockOutputStream.toByteArray(); } - private IterationResult withIterator( - CheckedFunction<DBIterator, IterationResult> func) throws IOException { - try (DBIterator iterator = levelDb.iterator()) { - return func.apply(iterator); - } - } - /** * CheckedFunction is akin to {@link java.util.function.Function} but * specifies an IOException. - * @param <T> Argument type. + * @param <T1> First argument type. + * @param <T2> Second argument type. * @param <R> Return type. */ @FunctionalInterface - public interface CheckedFunction<T, R> { - R apply(T t) throws IOException; + public interface CheckedFunction2<T1, T2, R> { + R apply(T1 t1, T2 t2) throws IOException; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java index a229ae7..4edc9a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java @@ -38,7 +38,7 @@ import java.util.Optional; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.protocol.proto.AliasMapProtocolProtos.*; -import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.CheckedFunction; +import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.CheckedFunction2; /** * InMemoryLevelDBAliasMapServer is the entry point from the Namenode into @@ -51,14 +51,15 @@ public class InMemoryLevelDBAliasMapServer implements InMemoryAliasMapProtocol, private static final Logger LOG = LoggerFactory .getLogger(InMemoryLevelDBAliasMapServer.class); - private final CheckedFunction<Configuration, InMemoryAliasMap> initFun; + private final CheckedFunction2<Configuration, String, InMemoryAliasMap> + initFun; private RPC.Server aliasMapServer; private Configuration conf; private InMemoryAliasMap aliasMap; private String blockPoolId; public InMemoryLevelDBAliasMapServer( - CheckedFunction<Configuration, InMemoryAliasMap> initFun, + CheckedFunction2<Configuration, String, InMemoryAliasMap> initFun, String blockPoolId) { this.initFun = initFun; this.blockPoolId = blockPoolId; @@ -127,7 +128,7 @@ public class InMemoryLevelDBAliasMapServer implements InMemoryAliasMapProtocol, public void setConf(Configuration conf) { this.conf = conf; try { - this.aliasMap = initFun.apply(conf); + this.aliasMap = initFun.apply(conf, blockPoolId); } catch (IOException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java index adad79a..6afe6bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java @@ -113,6 +113,11 @@ public class LevelDBFileRegionAliasMap } else { dbFile = new File(levelDBPath); } + if (createIfMissing && !dbFile.exists()) { + if (!dbFile.mkdirs()) { + throw new IOException("Unable to create " + dbFile); + } + } return factory.open(dbFile, options); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md index 7455044..01e7076 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md @@ -134,7 +134,7 @@ hadoop org.apache.hadoop.hdfs.server.namenode.FileSystemImage \ Assign ownership based on a custom `UGIResolver`, in LevelDB: ``` hadoop org.apache.hadoop.hdfs.server.namenode.FileSystemImage \ - -Ddfs.provided.aliasmap.leveldb.path=file:///path/to/leveldb/map/dingos.db \ + -Ddfs.provided.aliasmap.leveldb.path=/path/to/leveldb/map/dingos.db \ -b org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap \ -o file:///tmp/name \ -u CustomResolver \ @@ -180,7 +180,7 @@ Datanodes contact this alias map using the `org.apache.hadoop.hdfs.server.aliasm <property> <name>dfs.provided.aliasmap.inmemory.leveldb.dir</name> - <value>file:///path/to/leveldb/map/dingos.db</value> + <value>/path/to/leveldb/map/dingos.db</value> <description> The directory where the leveldb files will be kept </description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java index 6f1ff3e..432aefb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java @@ -44,14 +44,17 @@ import java.util.Optional; public class ITestInMemoryAliasMap { private InMemoryAliasMap aliasMap; private File tempDirectory; + private static String bpid = "bpid-0"; @Before public void setUp() throws Exception { Configuration conf = new Configuration(); - tempDirectory = Files.createTempDirectory("seagull").toFile(); + File temp = Files.createTempDirectory("seagull").toFile(); + tempDirectory = new File(temp, bpid); + tempDirectory.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, - tempDirectory.getAbsolutePath()); - aliasMap = InMemoryAliasMap.init(conf); + temp.getAbsolutePath()); + aliasMap = InMemoryAliasMap.init(conf, bpid); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java index f699055..0f57247 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java @@ -39,7 +39,7 @@ public class TestInMemoryAliasMap { nonExistingDirectory); assertThatExceptionOfType(IOException.class) - .isThrownBy(() -> InMemoryAliasMap.init(conf)).withMessage( + .isThrownBy(() -> InMemoryAliasMap.init(conf, "bpid")).withMessage( InMemoryAliasMap.createPathErrorMessage(nonExistingDirectory)); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java index a388199..61a1558 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -66,6 +66,8 @@ public class TestInMemoryLevelDBAliasMapClient { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); tempDir = Files.createTempDir(); + File levelDBDir = new File(tempDir, BPID); + levelDBDir.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDir.getAbsolutePath()); levelDBAliasMapServer = http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java index 8212b28..534bc36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java @@ -54,7 +54,7 @@ public class TestLevelDbMockAliasMapClient { aliasMapMock = mock(InMemoryAliasMap.class); when(aliasMapMock.getBlockPoolId()).thenReturn(bpid); levelDBAliasMapServer = new InMemoryLevelDBAliasMapServer( - config -> aliasMapMock, bpid); + (config, blockPoolID) -> aliasMapMock, bpid); conf = new Configuration(); int port = 9877; http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c3fec3/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java index 2917a34..1023616 100644 --- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java +++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java @@ -784,6 +784,8 @@ public class TestNameNodeProvidedImplementation { "localhost:32445"); File tempDirectory = Files.createTempDirectory("in-memory-alias-map").toFile(); + File leveDBPath = new File(tempDirectory, bpid); + leveDBPath.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDirectory.getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org