Repository: kylin Updated Branches: refs/heads/KYLIN-1356 [created] 7f08dab55
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java deleted file mode 100644 index 1546576..0000000 --- a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.jdbc; - -import java.io.File; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.io.FileUtils; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.webapp.WebAppContext; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - */ -public class ITJDBCDriverTest extends HBaseMetadataTestCase { - - private static Server server = null; - private static SystemPropertiesOverride sysPropsOverride = new SystemPropertiesOverride(); - - @BeforeClass - public static void beforeClass() throws Exception { - sysPropsOverride.override("spring.profiles.active", "testing"); - sysPropsOverride.override("catalina.home", "."); // resources/log4j.properties ref ${catalina.home} - staticCreateTestMetadata(); - startJetty(); - } - - @AfterClass - public static void afterClass() throws Exception { - stopJetty(); - staticCleanupTestMetadata(); - sysPropsOverride.restore(); - } - - protected static void stopJetty() throws Exception { - if (server != null) - server.stop(); - - File workFolder = new File("work"); - if (workFolder.isDirectory() && workFolder.exists()) { - FileUtils.deleteDirectory(workFolder); - } - } - - protected static void startJetty() throws Exception { - - server = new Server(7070); - - WebAppContext context = new WebAppContext(); - context.setDescriptor("./src/main/webapp/WEB-INF/web.xml"); - context.setResourceBase("./src/main/webapp"); - context.setContextPath("/kylin"); - context.setParentLoaderPriority(true); - - server.setHandler(context); - - server.start(); - - } - - protected Connection getConnection() throws Exception { - - Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance(); - Properties info = new Properties(); - info.put("user", "ADMIN"); - info.put("password", "KYLIN"); - Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info); - - return conn; - } - - @Test - public void testMetadata1() throws Exception { - - // check the JDBC API here: http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html - Connection conn = getConnection(); - - // test getSchemas(); - List<String> schemaList = Lists.newArrayList(); - DatabaseMetaData dbMetadata = conn.getMetaData(); - ResultSet resultSet = dbMetadata.getSchemas(); - while (resultSet.next()) { - String schema = resultSet.getString("TABLE_SCHEM"); - String catalog = resultSet.getString("TABLE_CATALOG"); - - System.out.println("Get schema: schema=" + schema + ", catalog=" + catalog); - schemaList.add(schema); - - } - - resultSet.close(); - Assert.assertTrue(schemaList.contains("DEFAULT")); - Assert.assertTrue(schemaList.contains("EDW")); - - // test getCatalogs(); - resultSet = dbMetadata.getCatalogs(); - - List<String> catalogList = Lists.newArrayList(); - while (resultSet.next()) { - String catalog = resultSet.getString("TABLE_CAT"); - - System.out.println("Get catalog: catalog=" + catalog); - catalogList.add(catalog); - - } - Assert.assertTrue(catalogList.size() > 0 && catalogList.contains("defaultCatalog")); - - /** //Disable the test on getTableTypes() as it is not ready - resultSet = dbMetadata.getTableTypes(); - - List<String> tableTypes = Lists.newArrayList(); - while (resultSet.next()) { - String type = resultSet.getString("TABLE_TYPE"); - - System.out.println("Get table type: type=" + type); - tableTypes.add(type); - - } - - Assert.assertTrue(tableTypes.size() > 0 && tableTypes.contains("TABLE")); - resultSet.close(); - - **/ - conn.close(); - } - - @Test - public void testMetadata2() throws Exception { - Connection conn = getConnection(); - - List<String> tableList = Lists.newArrayList(); - DatabaseMetaData dbMetadata = conn.getMetaData(); - ResultSet resultSet = dbMetadata.getTables(null, "%", "%", new String[] { "TABLE" }); - while (resultSet.next()) { - String schema = resultSet.getString("TABLE_SCHEM"); - String name = resultSet.getString("TABLE_NAME"); - - System.out.println("Get table: schema=" + schema + ", name=" + name); - tableList.add(schema + "." + name); - - } - - resultSet.close(); - Assert.assertTrue(tableList.contains("DEFAULT.TEST_KYLIN_FACT")); - - resultSet = dbMetadata.getColumns(null, "%", "TEST_KYLIN_FACT", "%"); - - List<String> columns = Lists.newArrayList(); - while (resultSet.next()) { - String name = resultSet.getString("COLUMN_NAME"); - String type = resultSet.getString("TYPE_NAME"); - - System.out.println("Get column: name=" + name + ", data_type=" + type); - columns.add(name); - - } - - Assert.assertTrue(columns.size() > 0 && columns.contains("CAL_DT")); - resultSet.close(); - conn.close(); - } - - @Test - public void testSimpleStatement() throws Exception { - Connection conn = getConnection(); - Statement statement = conn.createStatement(); - - statement.execute("select count(*) from test_kylin_fact"); - - ResultSet rs = statement.getResultSet(); - - Assert.assertTrue(rs.next()); - int result = rs.getInt(1); - - Assert.assertTrue(result > 0); - - rs.close(); - statement.close(); - conn.close(); - - } - - @Test - public void testPreparedStatement() throws Exception { - Connection conn = getConnection(); - - PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME"); - - statement.setString(1, "FP-GTC"); - - ResultSet rs = statement.executeQuery(); - - Assert.assertTrue(rs.next()); - - String format_name = rs.getString(1); - - Assert.assertTrue("FP-GTC".equals(format_name)); - - rs.close(); - statement.close(); - conn.close(); - - } - - @Test - public void testResultSet() throws Exception { - String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME "; - - Connection conn = getConnection(); - Statement statement = conn.createStatement(); - - statement.execute(sql); - - ResultSet rs = statement.getResultSet(); - - int count = 0; - while (rs.next()) { - count++; - String lstg = rs.getString(1); - double gmv = rs.getDouble(2); - int trans_count = rs.getInt(3); - - System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count); - } - - Assert.assertTrue(count > 0); - statement.close(); - rs.close(); - conn.close(); - - } - - private static class SystemPropertiesOverride { - HashMap<String, String> backup = new HashMap<String, String>(); - - public void override(String key, String value) { - backup.put(key, System.getProperty(key)); - System.setProperty(key, value); - } - - public void restore() { - for (String key : backup.keySet()) { - String value = backup.get(key); - if (value == null) - System.clearProperty(key); - else - System.setProperty(key, value); - } - backup.clear(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java deleted file mode 100644 index b2b0f22..0000000 --- a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.controller; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.rest.service.CubeService; -import org.apache.kylin.rest.service.ServiceTestBase; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * @author shaoshi - */ -//TODO: break this into unit tests -@Ignore("ServiceTestBase has been changed to local meta tests, thus not suitable for hive related tests") -public class ITTableControllerTest extends ServiceTestBase { - - private TableController tableController; - private CubeDescController cubeDescController; - - @Autowired - CubeService cubeService; - - @Before - public void setup() throws Exception { - super.setup(); - - tableController = new TableController(); - tableController.setCubeService(cubeService); - } - - @Test - public void testBasics() throws IOException { - List<TableDesc> tables = tableController.getHiveTables(true, "default"); - - Assert.assertTrue(tables != null && tables.size() > 0); - - TableDesc factTable = null; - for (TableDesc t : tables) { - if (t.getName().equalsIgnoreCase("test_kylin_fact")) { - factTable = t; - break; - } - } - Assert.assertNotNull(factTable); - - Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default"); - Assert.assertNotNull(loadResult); - - Assert.assertTrue(loadResult.get("result.loaded").length == 2); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java deleted file mode 100644 index df52664..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.common; - -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.apache.kylin.metadata.tuple.ITuple; -import org.apache.kylin.metadata.tuple.ITupleIterator; -import org.apache.kylin.storage.IStorageQuery; -import org.apache.kylin.storage.StorageContext; -import org.apache.kylin.storage.StorageFactory; -import org.apache.kylin.storage.cache.StorageMockUtils; -import org.apache.kylin.storage.exception.ScanOutOfLimitException; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -public class ITStorageTest extends HBaseMetadataTestCase { - - private IStorageQuery storageEngine; - private CubeInstance cube; - private StorageContext context; - - @BeforeClass - public static void setupResource() throws Exception { - } - - @AfterClass - public static void tearDownResource() { - } - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - - CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); - cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY"); - Assert.assertNotNull(cube); - storageEngine = StorageFactory.createQuery(cube); - String url = KylinConfig.getInstanceFromEnv().getStorageUrl(); - context = new StorageContext(); - context.setConnUrl(url); - } - - @After - public void tearDown() throws Exception { - this.cleanupTestMetadata(); - } - - @Test(expected = ScanOutOfLimitException.class) - @Ignore - public void testScanOutOfLimit() { - context.setThreshold(1); - List<TblColRef> groups = StorageMockUtils.buildGroups(); - List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); - - search(groups, aggregations, null, context); - } - - @Test - public void test01() { - List<TblColRef> groups = StorageMockUtils.buildGroups(); - List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations(); - TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0)); - - int count = search(groups, aggregations, filter, context); - assertTrue(count > 0); - } - - /* - @Test - public void test02() { - List<TblColRef> groups = buildGroups(); - List<FunctionDesc> aggregations = buildAggregations(); - TupleFilter filter = buildFilter2(groups.get(1)); - - int count = search(groups, aggregations, filter, context); - assertTrue(count > 0); - } - - @Test - public void test03() { - List<TblColRef> groups = buildGroups(); - List<FunctionDesc> aggregations = buildAggregations(); - TupleFilter filter = buildAndFilter(groups); - - int count = search(groups, aggregations, filter, context); - assertTrue(count > 0); - } - - @Test - public void test04() { - List<TblColRef> groups = buildGroups(); - List<FunctionDesc> aggregations = buildAggregations(); - TupleFilter filter = buildOrFilter(groups); - - int count = search(groups, aggregations, filter, context); - assertTrue(count > 0); - } - - @Test - public void test05() { - List<TblColRef> groups = buildGroups(); - List<FunctionDesc> aggregations = buildAggregations(); - - int count = search(groups, aggregations, null, context); - assertTrue(count > 0); - } - */ - private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) { - int count = 0; - ITupleIterator iterator = null; - try { - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>()); - iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations)); - while (iterator.hasNext()) { - ITuple tuple = iterator.next(); - System.out.println("Tuple = " + tuple); - count++; - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (iterator != null) - iterator.close(); - } - return count; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java deleted file mode 100644 index e9812da..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.ii; - -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.RawTableRecord; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecord; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.storage.hbase.HBaseConnection; -import org.apache.kylin.storage.hbase.cube.v1.HBaseClientKVIterator; -import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * @author yangli9 - */ -public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase { - - IIInstance ii; - IISegment seg; - HConnection hconn; - - TableRecordInfo info; - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - - this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join"); - this.seg = ii.getFirstSegment(); - - Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - hconn = HConnectionManager.createConnection(hconf); - - this.info = new TableRecordInfo(seg); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testLoad() throws Exception { - - String tableName = seg.getStorageLocationIdentifier(); - IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); - - List<Slice> slices = Lists.newArrayList(); - HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES); - try { - for (Slice slice : codec.decodeKeyValue(kvIterator)) { - slices.add(slice); - } - } finally { - kvIterator.close(); - } - - List<TableRecord> records = iterateRecords(slices); - //dump(records); - System.out.println("table name:" + tableName + " has " + records.size() + " records"); - } - - private List<TableRecord> iterateRecords(List<Slice> slices) { - List<TableRecord> records = Lists.newArrayList(); - for (Slice slice : slices) { - for (RawTableRecord rec : slice) { - records.add(new TableRecord((RawTableRecord) rec.clone(), info)); - } - } - return records; - } - - @SuppressWarnings("unused") - private void dump(Iterable<TableRecord> records) { - for (TableRecord rec : records) { - byte[] x = rec.getBytes(); - String y = BytesUtil.toReadableText(x); - System.out.println(y); - System.out.println(); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java deleted file mode 100644 index 2a0adc3..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.steps; - -import java.io.File; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; - -/** - * @author ysong1 - */ -public class HBaseMetadataTestCase extends AbstractKylinTestCase { - - static { - try { - ClassUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Override - public void createTestMetadata() throws Exception { - staticCreateTestMetadata(); - } - - @Override - public void cleanupTestMetadata() { - staticCleanupTestMetadata(); - } - - public static void staticCreateTestMetadata() throws Exception { - staticCreateTestMetadata(SANDBOX_TEST_DATA); - } - - public static void staticCreateTestMetadata(String kylinConfigFolder) { - - KylinConfig.destoryInstance(); - - if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null) - System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHBaseResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHBaseResourceStoreTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHBaseResourceStoreTest.java deleted file mode 100644 index c21bf78..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHBaseResourceStoreTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.steps; - -import static org.junit.Assert.*; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.storage.hbase.HBaseConnection; -import org.apache.kylin.storage.hbase.HBaseResourceStore; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testHBaseStore() throws Exception { - testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv())); - } - - @Test - public void testHBaseStoreWithLargeCell() throws Exception { - String path = "/cube/_test_large_cell.json"; - String largeContent = "THIS_IS_A_LARGE_CELL"; - StringEntity content = new StringEntity(largeContent); - KylinConfig config = KylinConfig.getInstanceFromEnv(); - int origSize = config.getHBaseKeyValueSize(); - ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - - try { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); - - store.deleteResource(path); - - store.putResource(path, content, StringEntity.serializer); - assertTrue(store.exists(path)); - StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer); - assertEquals(content, t); - - Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path); - Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - FileSystem fileSystem = FileSystem.get(hconf); - assertTrue(fileSystem.exists(redirectPath)); - - FSDataInputStream in = fileSystem.open(redirectPath); - assertEquals(largeContent, in.readUTF()); - in.close(); - - store.deleteResource(path); - } finally { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize); - store.deleteResource(path); - } - } - - void testAStore(ResourceStore store) throws IOException { - String dir1 = "/cube"; - String path1 = "/cube/_test.json"; - StringEntity content1 = new StringEntity("anything"); - String dir2 = "/table"; - String path2 = "/table/_test.json"; - StringEntity content2 = new StringEntity("something"); - - // cleanup legacy if any - store.deleteResource(path1); - store.deleteResource(path2); - - StringEntity t; - - // put/get - store.putResource(path1, content1, StringEntity.serializer); - assertTrue(store.exists(path1)); - t = store.getResource(path1, StringEntity.class, StringEntity.serializer); - assertEquals(content1, t); - - store.putResource(path2, content2, StringEntity.serializer); - assertTrue(store.exists(path2)); - t = store.getResource(path2, StringEntity.class, StringEntity.serializer); - assertEquals(content2, t); - - // overwrite - t.str = "new string"; - store.putResource(path2, t, StringEntity.serializer); - - // write conflict - try { - t.setLastModified(t.getLastModified() - 1); - store.putResource(path2, t, StringEntity.serializer); - fail("write conflict should trigger IllegalStateException"); - } catch (IllegalStateException e) { - // expected - } - - // list - ArrayList<String> list; - - list = store.listResources(dir1); - assertTrue(list.contains(path1)); - assertTrue(list.contains(path2) == false); - - list = store.listResources(dir2); - assertTrue(list.contains(path2)); - assertTrue(list.contains(path1) == false); - - list = store.listResources("/"); - assertTrue(list.contains(dir1)); - assertTrue(list.contains(dir2)); - assertTrue(list.contains(path1) == false); - assertTrue(list.contains(path2) == false); - - list = store.listResources(path1); - assertNull(list); - list = store.listResources(path2); - assertNull(list); - - // delete/exist - store.deleteResource(path1); - assertTrue(store.exists(path1) == false); - list = store.listResources(dir1); - assertTrue(list == null || list.contains(path1) == false); - - store.deleteResource(path2); - assertTrue(store.exists(path2) == false); - list = store.listResources(dir2); - assertTrue(list == null || list.contains(path2) == false); - } - - public static class StringEntity extends RootPersistentEntity { - - static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() { - @Override - public void serialize(StringEntity obj, DataOutputStream out) throws IOException { - out.writeUTF(obj.str); - } - - @Override - public StringEntity deserialize(DataInputStream in) throws IOException { - String str = in.readUTF(); - return new StringEntity(str); - } - }; - - String str; - - public StringEntity(String str) { - this.str = str; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((str == null) ? 0 : str.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) - return true; - if (!(obj instanceof StringEntity)) - return false; - return StringUtils.equals(this.str, ((StringEntity) obj).str); - } - - @Override - public String toString() { - return str; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java deleted file mode 100644 index 25ac2c6..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ITHdfsOpsTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.steps; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.HadoopUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - */ -public class ITHdfsOpsTest extends HBaseMetadataTestCase { - - FileSystem fileSystem; - - @Before - public void setup() throws Exception { - - this.createTestMetadata(); - - Configuration hconf = HadoopUtil.getCurrentConfiguration(); - - fileSystem = FileSystem.get(hconf); - } - - @Test - public void TestPath() throws IOException { - String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); - Path coprocessorDir = new Path(hdfsWorkingDirectory, "test"); - fileSystem.mkdirs(coprocessorDir); - - Path newFile = new Path(coprocessorDir, "test_file"); - newFile = newFile.makeQualified(fileSystem.getUri(), null); - FSDataOutputStream stream = fileSystem.create(newFile); - stream.write(new byte[] { 0, 1, 2 }); - stream.close(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/7f08dab5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index a003d6a..3bf5f00 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceTool; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.common.util.HBaseMetadataTestCase; /** * This is a helper class for developer to directly manipulate the metadata store in sandbox
