Repository: hadoop Updated Branches: refs/heads/HDFS-7240 c2d08e601 -> 5d0f54514
HDFS-12155. Ozone : add RocksDB support to DEBUG CLI. Contributed by Chen Liang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d0f5451 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d0f5451 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d0f5451 Branch: refs/heads/HDFS-7240 Commit: 5d0f545143a52c97b2d02cfd689416994b85a932 Parents: c2d08e6 Author: Weiwei Yang <w...@apache.org> Authored: Wed Jul 26 12:59:25 2017 +0800 Committer: Weiwei Yang <w...@apache.org> Committed: Wed Jul 26 12:59:25 2017 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 15 +++--- .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 24 +++++++-- .../hadoop/ozone/scm/TestContainerSQLCli.java | 54 ++++++++++++++------ 3 files changed, 66 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d0f5451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java index 59fe2ae..30a2d22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.OzoneAclInfo; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; @@ -75,6 +76,7 @@ public class SQLCLI extends Configured implements Tool { private Options options; private BasicParser parser; private final Charset encoding = Charset.forName("UTF-8"); + private final OzoneConfiguration conf; // for container.db private static final String CREATE_CONTAINER_INFO = @@ -199,9 +201,10 @@ public class SQLCLI extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(SQLCLI.class); - public SQLCLI() { + public SQLCLI(OzoneConfiguration conf) { this.options = getOptions(); this.parser = new BasicParser(); + this.conf = conf; } @SuppressWarnings("static-access") @@ -468,7 +471,7 @@ public class SQLCLI extends Configured implements Tool { LOG.info("Create tables for sql container db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_CONTAINER_INFO); executeSQL(conn, CREATE_CONTAINER_MEMBERS); @@ -547,7 +550,7 @@ public class SQLCLI extends Configured implements Tool { LOG.info("Create tables for sql block db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_BLOCK_CONTAINER); @@ -594,7 +597,7 @@ public class SQLCLI extends Configured implements Tool { LOG.info("Create table for sql node pool db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_NODE_POOL); executeSQL(conn, CREATE_DATANODE_INFO); @@ -645,7 +648,7 @@ public class SQLCLI extends Configured implements Tool { LOG.info("Create table for open container db."); File dbFile = dbPath.toFile(); try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setDbFile(dbFile).build(); + .setConf(conf).setDbFile(dbFile).build(); Connection conn = connectDB(outPath.toString())) { executeSQL(conn, CREATE_OPEN_CONTAINER); @@ -671,7 +674,7 @@ public class SQLCLI extends Configured implements Tool { } public static void main(String[] args) { - Tool shell = new SQLCLI(); + Tool shell = new SQLCLI(new OzoneConfiguration()); int res = 0; try { ToolRunner.run(shell, args); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d0f5451/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java index 97979ed..6a2b0f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java @@ -30,9 +30,9 @@ import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.IOException; import java.io.OutputStream; @@ -44,6 +44,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -57,6 +58,7 @@ import static org.junit.Assert.assertTrue; /** * This class tests the CLI that transforms ksm.db into SQLite DB files. */ +@RunWith(Parameterized.class) public class TestKSMSQLCli { private static MiniOzoneCluster cluster = null; private static StorageHandler storageHandler; @@ -76,8 +78,19 @@ public class TestKSMSQLCli { private static String keyName2 = "key2"; private static String keyName3 = "key3"; - @Rule - public ExpectedException exception = ExpectedException.none(); + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private static String metaStoreType; + + public TestKSMSQLCli(String type) { + metaStoreType = type; + } /** * Create a MiniDFSCluster for testing. @@ -142,7 +155,8 @@ public class TestKSMSQLCli { @Before public void init() throws Exception { - cli = new SQLCLI(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d0f5451/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java index 1e6f5f7..6ffbb4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java @@ -33,9 +33,11 @@ import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.scm.container.common.helpers.Pipeline; import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.nio.file.Files; import java.nio.file.Paths; @@ -45,6 +47,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -59,27 +63,43 @@ import static org.junit.Assert.assertTrue; /** * This class tests the CLI that transforms container into SQLite DB files. */ +@RunWith(Parameterized.class) public class TestContainerSQLCli { + + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private static String metaStoreType; + + public TestContainerSQLCli(String type) { + metaStoreType = type; + } + private static SQLCLI cli; - private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf; - private static StorageContainerLocationProtocolClientSideTranslatorPB + private MiniOzoneCluster cluster; + private OzoneConfiguration conf; + private StorageContainerLocationProtocolClientSideTranslatorPB storageContainerLocationClient; - private static ContainerMapping mapping; - private static NodeManager nodeManager; - private static BlockManagerImpl blockManager; + private ContainerMapping mapping; + private NodeManager nodeManager; + private BlockManagerImpl blockManager; - private static Pipeline pipeline1; - private static Pipeline pipeline2; + private Pipeline pipeline1; + private Pipeline pipeline2; - private static HashMap<String, String> blockContainerMap; + private HashMap<String, String> blockContainerMap; private final static long DEFAULT_BLOCK_SIZE = 4 * KB; - @BeforeClass - public static void init() throws Exception { + @Before + public void setup() throws Exception { long datanodeCapacities = 3 * OzoneConsts.TB; blockContainerMap = new HashMap<>(); @@ -138,11 +158,13 @@ public class TestContainerSQLCli { mapping.close(); nodeManager.close(); - cli = new SQLCLI(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); + } - @AfterClass - public static void shutdown() throws InterruptedException { + @After + public void shutdown() throws InterruptedException { IOUtils.cleanup(null, storageContainerLocationClient, cluster); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
