Repository: lens Updated Branches: refs/heads/master dfd332902 -> 3d1b48c44
LENS-1038: Streaming and other session related tests Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3d1b48c4 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3d1b48c4 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3d1b48c4 Branch: refs/heads/master Commit: 3d1b48c445ccec9619c6e1da753f72f8d735a26d Parents: dfd3329 Author: Archana H <archana.h...@gmail.com> Authored: Thu May 12 14:59:56 2016 +0530 Committer: Puneet <puneet.gu...@inmobi.com> Committed: Thu May 12 14:59:56 2016 +0530 ---------------------------------------------------------------------- .../regression/core/constants/DriverConfig.java | 1 - .../lens/regression/ITStreamingTests.java | 166 +++++++++++++++++++ .../apache/lens/regression/SessionTests.java | 158 ++++++++++++++++++ 3 files changed, 324 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/3d1b48c4/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java index da13715..c0fc15a 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java @@ -36,6 +36,5 @@ public class DriverConfig { MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY; public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties. JDBC_POOL_MAX_SIZE.getConfigKey(); - } http://git-wip-us.apache.org/repos/asf/lens/blob/3d1b48c4/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java new file mode 100644 index 0000000..d942e3c --- /dev/null +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.regression; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.HashMap; + +import javax.ws.rs.client.WebTarget; +import javax.xml.bind.JAXBException; + +import org.apache.lens.api.query.*; +import org.apache.lens.regression.core.constants.QueryInventory; +import org.apache.lens.regression.core.helpers.*; +import org.apache.lens.regression.core.testHelper.BaseTestClass; +import org.apache.lens.regression.util.Util; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; + +import org.apache.log4j.Logger; + +import org.testng.Assert; +import org.testng.annotations.*; + +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.SftpException; + + +public class ITStreamingTests extends BaseTestClass { + + private WebTarget servLens; + private String sessionHandleString; + + LensServerHelper lens = getLensServerHelper(); + MetastoreHelper mHelper = getMetastoreHelper(); + SessionHelper sHelper = getSessionHelper(); + QueryHelper qHelper = getQueryHelper(); + + private static Logger logger = Logger.getLogger(ITStreamingTests.class); + + String lensSiteConfPath = lens.getServerDir() + "/conf/lens-site.xml"; + + @BeforeClass(alwaysRun = true) + public void initialize() throws IOException, JAXBException, LensException { + servLens = ServiceManagerHelper.init(); + } + + @BeforeMethod(alwaysRun = true) + public void setUp(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + logger.info("Creating a new Session"); + sessionHandleString = lens.openSession(lens.getCurrentDB()); + } + + @AfterMethod(alwaysRun = true) + public void closeSession() throws Exception { + logger.info("Closing Session"); + lens.closeSession(); + } + + @DataProvider(name="query_provider") + public Object[][] queryProvider(){ + Object[][] query = { {QueryInventory.JDBC_DIM_QUERY, new Integer(2)}, + {QueryInventory.HIVE_CUBE_QUERY, new Integer(8)}, + {QueryInventory.HIVE_DIM_QUERY, new Integer(9)}, + }; + return query; + } + + + @BeforeGroups("large_purge_interval") + public void setLargePurgerInterval() throws Exception { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.PURGE_INTERVAL, "1000000"); + Util.changeConfig(map, lensSiteConfPath); + lens.restart(); + } + + @AfterGroups("large_purge_interval") + public void restoreConfig() throws SftpException, JSchException, InterruptedException, LensException, IOException { + Util.changeConfig(lensSiteConfPath); + lens.restart(); + } + + + @Test(enabled=true, groups= "large_purge_interval", dataProvider="query_provider") + public void testQueryResultStreaming(String query, int expectedNumRows) throws Exception { + + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + + //setting high timeout value so that query completes within timeout period + QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout(query, "100000").getData(); + + InMemoryQueryResult inmemoryResult = (InMemoryQueryResult) qhr.getResult(); + Assert.assertNotNull(inmemoryResult); + Assert.assertEquals(inmemoryResult.getRows().size(), expectedNumRows); + + LensQuery lq = qHelper.waitForCompletion(qhr.getQueryHandle()); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + QueryResult queryRes = qHelper.getResultSet(qhr.getQueryHandle()); + + Assert.assertTrue(queryRes instanceof PersistentQueryResult); + + PersistentQueryResult persistResult = (PersistentQueryResult) queryRes; + Assert.assertNotNull(persistResult); + Assert.assertEquals(persistResult.getNumRows().intValue(), expectedNumRows); + } + + + @Test(enabled=true, groups= "large_purge_interval") + public void testQueryRunningAfterTimeout() throws Exception { + + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + + QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout( + QueryInventory.getSleepQuery("10"), "1000").getData(); + InMemoryQueryResult inmemoryResult = (InMemoryQueryResult) qhr.getResult(); + Assert.assertNull(inmemoryResult); + + LensQuery lensQuery = qHelper.waitForCompletion(qhr.getQueryHandle()); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + PersistentQueryResult persistResult = (PersistentQueryResult)qHelper.getResultSet(qhr.getQueryHandle()); + Assert.assertNotNull(persistResult); + Assert.assertEquals(persistResult.getNumRows().intValue(), 7); + } + + + @Test(enabled=true, groups= "large_purge_interval") + public void resultMoreThanMaxPrefetchRows() throws Exception { + + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + sHelper.setAndValidateParam(LensConfConstants.PREFETCH_INMEMORY_RESULTSET_ROWS, "5"); + + //Result of this query has 8 rows + QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout(QueryInventory. + HIVE_CUBE_QUERY, "100000").getData(); + + //Since query has more rows than prefetch limit, ResultSet should be PersistentQueryResult + Assert.assertFalse(qhr.getResult() instanceof InMemoryQueryResult); + + PersistentQueryResult result = (PersistentQueryResult) qhr.getResult(); + Assert.assertNotNull(result); + Assert.assertEquals(result.getNumRows().intValue(), 8); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/3d1b48c4/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java b/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java new file mode 100644 index 0000000..8b53897 --- /dev/null +++ b/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.regression; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + +import javax.xml.bind.JAXBException; + +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.regression.core.constants.DriverConfig; +import org.apache.lens.regression.core.constants.QueryInventory; +import org.apache.lens.regression.core.constants.SessionURL; +import org.apache.lens.regression.core.helpers.*; +import org.apache.lens.regression.core.testHelper.BaseTestClass; +import org.apache.lens.regression.core.type.MapBuilder; +import org.apache.lens.regression.util.Util; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; + +import org.apache.log4j.Logger; + +import org.testng.Assert; +import org.testng.annotations.*; + +import com.jcraft.jsch.JSchException; + + +public class SessionTests extends BaseTestClass { + + private WebTarget servLens; + private String sessionHandleString; + + LensServerHelper lens = getLensServerHelper(); + MetastoreHelper mHelper = getMetastoreHelper(); + SessionHelper sHelper = getSessionHelper(); + QueryHelper qHelper = getQueryHelper(); + + private static Logger logger = Logger.getLogger(SessionTests.class); + + @BeforeClass(alwaysRun = true) + public void initialize() throws IOException, JSchException, JAXBException, LensException { + servLens = ServiceManagerHelper.init(); + } + + @BeforeMethod(alwaysRun = true) + public void setUp(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + logger.info("Creating a new Session"); + sessionHandleString = lens.openSession(lens.getCurrentDB()); + } + + @AfterMethod(alwaysRun = true) + public void closeSession() throws Exception { + logger.info("Closing Session"); + lens.closeSession(); + } + + + @Test(enabled = true) + public void testServerConfExposureInSession() throws Exception { + + // conf : lens-site.xml + MapBuilder query1 = new MapBuilder("sessionid", sessionHandleString, "key", LensConfConstants.SERVER_DB_JDBC_PASS); + Response response1 = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query1); + Assert.assertEquals(response1.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + //Driver conf : jdbc-driver.xml + MapBuilder query2 = new MapBuilder("sessionid", sessionHandleString, "key", "lens.driver.jdbc.db.user"); + Response response2 = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query2); + Assert.assertEquals(response2.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + } + + + // LENS-760. Check for only running as queued is not fixed. + @Test(enabled = true) + public void testRunningQueryContinuationOnSessionClose() throws Exception { + + String session = sHelper.openNewSession("test", "test", lens.getCurrentDB()); + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + String sleepQuery = QueryInventory.getSleepQuery("5"); + + for(int i=1; i<=5; i++){ + handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, session).getData()); + } + + Thread.sleep(3000); + + List<QueryHandle> running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString, null, null); + sHelper.closeNewSession(session); + Assert.assertTrue(running.size() > 0); + logger.info("Running query count " + running.size()); + + for(QueryHandle handle : running){ + LensQuery lq = qHelper.waitForCompletion(handle); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + } + + // Fails. Bug : LENS-904 + // Check for query continuation on session close. + @Test(enabled = false) + public void testQueryContinuationOnSessionClose() throws Exception { + + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1"); + String expHiveDriverConf = lens.getServerDir() + "/conf/drivers/hive/hive1/hivedriver-site.xml"; + + try { + Util.changeConfig(map, expHiveDriverConf); + lens.restart(); + + String session = sHelper.openNewSession("test", "test", lens.getCurrentDB()); + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + String sleepQuery = QueryInventory.getSleepQuery("5"); + + for (int i = 1; i <= 5; i++) { + handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, session).getData()); + } + + sHelper.closeNewSession(session); + + for (QueryHandle handle : handleList) { + LensQuery lq = qHelper.waitForCompletion(handle); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + } finally { + Util.changeConfig(expHiveDriverConf); + lens.restart(); + } + } +}
