Repository: kylin Updated Branches: refs/heads/master 5d2e6c10c -> 4dce0cf08
KYLIN-2417 Compare the performance between HDFSMetaStore and HBaseMetaStore Signed-off-by: Li Yang <liy...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65acd70e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65acd70e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65acd70e Branch: refs/heads/master Commit: 65acd70ed4ff1a6f16a1ede95c1c9b85dc2f5578 Parents: 5d2e6c1 Author: xiefan46 <958034...@qq.com> Authored: Wed Jan 25 17:07:43 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Tue Feb 21 10:04:02 2017 +0800 ---------------------------------------------------------------------- .../common/persistence/ResourceStoreTest.java | 42 +++++++++++++++++++- .../storage/hdfs/ITHDFSResourceStoreTest.java | 19 +++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/65acd70e/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java index ddaf481..79a0c30 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java @@ -30,17 +30,34 @@ import java.util.List; import java.util.NavigableSet; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Be called by LocalFileResourceStoreTest and ITHBaseResourceStoreTest. + * Be called by LocalFileResourceStoreTest, ITHBaseResourceStoreTest and ITHDFSResourceStoreTest. */ public class ResourceStoreTest { + private static final Logger logger = LoggerFactory.getLogger(ResourceStoreTest.class); + + private static final String PERFORMANCE_TEST_ROOT_PATH = "/performance"; + + private static final int TEST_RESOURCE_COUNT = 1000; + public static void testAStore(ResourceStore store) throws IOException { testBasics(store); testGetAllResources(store); } + public static void testPerformance(ResourceStore store) throws IOException { + logger.info("Test basic functions"); + testAStore(store); + logger.info("Basic function ok. Start to test performance for class : " + store.getClass()); + logger.info("Write metadata time : " + testWritePerformance(store)); + logger.info("Read metadata time " + testReadPerformance(store)); + logger.info("Performance test end. Class : " + store.getClass()); + } + private static void testGetAllResources(ResourceStore store) throws IOException { final String folder = "/testFolder"; List<StringEntity> result; @@ -144,6 +161,29 @@ public class ResourceStoreTest { assertTrue(list == null || list.contains(path2) == false); } + private static long testWritePerformance(ResourceStore store) throws IOException { + store.deleteResource(PERFORMANCE_TEST_ROOT_PATH); + StringEntity content = new StringEntity("something"); + long startTime = System.currentTimeMillis(); + for (int i = 0; i < TEST_RESOURCE_COUNT; i++) { + String resourcePath = PERFORMANCE_TEST_ROOT_PATH + "/res_" + i; + store.putResource(resourcePath, content, 0, StringEntity.serializer); + } + return System.currentTimeMillis() - startTime; + } + + private static long testReadPerformance(ResourceStore store) throws IOException { + long startTime = System.currentTimeMillis(); + int step = 0; //avoid compiler optimization + for (int i = 0; i < TEST_RESOURCE_COUNT; i++) { + String resourcePath = PERFORMANCE_TEST_ROOT_PATH + "/res_" + i; + StringEntity t = store.getResource(resourcePath, StringEntity.class, StringEntity.serializer); + step |= t.toString().length(); + } + logger.info("step : " + step); + return System.currentTimeMillis() - startTime; + } + @SuppressWarnings("serial") public static class StringEntity extends RootPersistentEntity { http://git-wip-us.apache.org/repos/asf/kylin/blob/65acd70e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java index aa5a104..534839f 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java @@ -27,8 +27,10 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStoreTest; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.common.util.HadoopUtil; +import org.apache.kylin.storage.hbase.HBaseResourceStore; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { @@ -84,4 +86,21 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { assertTrue(fs.exists(new Path(path))); } + @Ignore + @Test + public void performanceTest() throws Exception{ + //test hdfs performance + String oldUrl = kylinConfig.getMetadataUrl(); + kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hdfs"); + HDFSResourceStore store = new HDFSResourceStore(kylinConfig); + ResourceStoreTest.testPerformance(store); + kylinConfig.setProperty("kylin.metadata.url", oldUrl); + + //test hbase + oldUrl = kylinConfig.getMetadataUrl(); + kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hbase"); + HBaseResourceStore store2 = new HBaseResourceStore(kylinConfig); + ResourceStoreTest.testPerformance(store2); + kylinConfig.setProperty("kylin.metadata.url", oldUrl); + } }
