Repository: lens Updated Branches: refs/heads/master d6b121692 -> 59c2f8b11
LENS-1020: Add server config tests to regression Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/59c2f8b1 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/59c2f8b1 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/59c2f8b1 Branch: refs/heads/master Commit: 59c2f8b11d1bfef58865e306b9b589c0ee6e0175 Parents: d6b1216 Author: Archana H <archana.h...@gmail.com> Authored: Tue May 10 14:59:03 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Tue May 10 14:59:03 2016 +0530 ---------------------------------------------------------------------- lens-regression/pom.xml | 5 + .../regression/core/constants/DriverConfig.java | 3 + .../core/constants/QueryInventory.java | 9 +- .../core/helpers/LensServerHelper.java | 5 +- .../regression/core/helpers/QueryHelper.java | 9 +- .../regression/core/helpers/SessionHelper.java | 2 +- .../apache/lens/regression/util/AssertUtil.java | 1 + .../org/apache/lens/regression/util/Util.java | 41 +- .../lens/regression/client/ITListQueryTest.java | 4 +- .../regression/client/ITPreparedQueryTests.java | 22 +- .../lens/regression/client/ITQueryApiTests.java | 10 +- .../lens/regression/client/KillQueryTests.java | 4 +- .../regression/client/SessionResourceTests.java | 6 +- .../regression/config/ITServerConfigTests.java | 601 +++++++++++++++++++ 14 files changed, 661 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/pom.xml ---------------------------------------------------------------------- diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml index 08a7ece..b9841e8 100644 --- a/lens-regression/pom.xml +++ b/lens-regression/pom.xml @@ -49,6 +49,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.lens</groupId> + <artifactId>lens-driver-jdbc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>com.jcraft</groupId> <artifactId>jsch</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java index 5a2df11..da13715 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java @@ -19,6 +19,7 @@ package org.apache.lens.regression.core.constants; +import org.apache.lens.driver.jdbc.JDBCDriverConfConstants; import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory; public class DriverConfig { @@ -33,6 +34,8 @@ public class DriverConfig { MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY; public static final String QUEUE_MAX_CONCURRENT = MaxConcurrentDriverQueriesConstraintFactory. MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY; + public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties. + JDBC_POOL_MAX_SIZE.getConfigKey(); } http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java index 869a96b..8de3994 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java @@ -39,7 +39,11 @@ public class QueryInventory { private QueryInventory() { + } + public static String getSleepQuery(String time){ + String query = String.format(SLEEP_QUERY_TIME, time); + return query; } public static final String QUERY = "cube select id,name from sample_dim where name != 'first'"; @@ -70,8 +74,7 @@ public class QueryInventory { public static final String SLEEP_FUNCTION = "CREATE TEMPORARY FUNCTION sleep AS 'hive.udf.SampleUdf'"; public static final String SLEEP_QUERY = "cube select sleep(name) from sample_dim where name != 'first'"; - - // public static final String SLEEP_QUERY = "cube select sleep(name,20000) from sample_dim where name != 'first'"; + public static final String SLEEP_QUERY_TIME = "cube select sleepTime(name,%s) from sample_dim where name != 'first'"; public static final String NO_CUBE_KEYWORD_QUERY = "select sample_dim_chain.name, measure4 from sample_cube where " + "time_range_in(dt, '2014-06-24-23', '2014-06-25-00')"; @@ -79,3 +82,5 @@ public class QueryInventory { public static final String QUOTE_QUERY = "cube select id,name from sample_dim2 where name != 'first\\'s'"; } + + http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java index 2f8b9c0..8019182 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java @@ -50,14 +50,15 @@ public class LensServerHelper extends ServiceManagerHelper { int counter = 0; Util.runRemoteCommand("bash /usr/local/lens/server/bin/lens-ctl stop"); Util.runRemoteCommand("bash /usr/local/lens/server/bin/lens-ctl start"); + Response response = this.exec("get", "", servLens, null, null, MediaType.TEXT_PLAIN_TYPE, MediaType.TEXT_PLAIN); while (response == null && counter < 40) { - Thread.sleep(5000); log.info("Waiting for Lens server to come up "); + Thread.sleep(1000); response = this.exec("get", "", servLens, null, null, MediaType.TEXT_PLAIN_TYPE, MediaType.TEXT_PLAIN); - log.info("Response:{}", response); counter++; } + AssertUtil.assertSucceededResponse(response); } } http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java index 2b4a750..8bd5290 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java @@ -299,13 +299,20 @@ public class QueryHelper extends ServiceManagerHelper { * @return the query Result */ - public QueryResult getResultSet(QueryHandle queryHandle, String fromIndex, String fetchSize, + public Response getResultSetResponse(QueryHandle queryHandle, String fromIndex, String fetchSize, String sessionHandleString) throws LensException { MapBuilder query = new MapBuilder("sessionid", sessionHandleString, "fromindex", fromIndex, "fetchsize", fetchSize); Response response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", servLens, null, query, MediaType.APPLICATION_XML_TYPE, MediaType.APPLICATION_XML, null); + return response; + } + + public QueryResult getResultSet(QueryHandle queryHandle, String fromIndex, String fetchSize, + String sessionHandleString) throws LensException { + + Response response = getResultSetResponse(queryHandle, fromIndex, fetchSize, sessionHandleString); AssertUtil.assertSucceededResponse(response); QueryResult result = response.readEntity(new GenericType<QueryResult>(){}); return result; http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java index 4918882..f7ec3e1 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java @@ -119,7 +119,7 @@ public class SessionHelper extends ServiceManagerHelper { MapBuilder query = new MapBuilder("sessionid", sessionHandleString); Response response = this.exec("delete", SessionURL.SESSION_BASE_URL, servLens, null, query, null, outputMediaType, null); - AssertUtil.assertSucceededResponse(response); + AssertUtil.assertSucceeded(response); log.info("Closed Session : {}", sessionHandleString); } http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java index ea36858..c4763aa 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java @@ -41,6 +41,7 @@ public class AssertUtil { * @param response Response * @throws LensException */ + public static void assertSucceeded(Response response) throws LensException { if (response.getStatus() != 200) { throw new LensException("Status code should be 200"); http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java index 6c3d234..1a538da 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java @@ -62,10 +62,9 @@ import lombok.extern.slf4j.Slf4j; public class Util { private static final String PROPERTY_FILE = "lens.properties"; - private static String localFilePath = "src/test/resources/"; + private static String localFilePath = "lens-regression/target/"; private static String localFile; private static String backupFile; - private static String remoteFile; private Util() { @@ -232,22 +231,16 @@ public class Util { } public static void changeConfig(HashMap<String, String> map, String remotePath) throws Exception { - String fileName; - remoteFile = remotePath; - Path p = Paths.get(remoteFile); - - fileName = p.getFileName().toString(); + Path p = Paths.get(remotePath); + String fileName = p.getFileName().toString(); backupFile = localFilePath + "backup-" + fileName; localFile = localFilePath + fileName; - log.info("Copying " + remoteFile + " to " + localFile); - remoteFile("get", remoteFile, localFile); - File locfile = new File(localFile); - File remfile = new File(backupFile); - Files.copy(locfile.toPath(), remfile.toPath(), REPLACE_EXISTING); - - DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + log.info("Copying " + remotePath + " to " + localFile); + remoteFile("get", remotePath, localFile); + Files.copy(new File(localFile).toPath(), new File(backupFile).toPath(), REPLACE_EXISTING); + + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document doc = docBuilder.parse(new FileInputStream(localFile)); doc.normalize(); @@ -256,8 +249,7 @@ public class Util { Element rootElement = (Element) root; NodeList property = rootElement.getElementsByTagName("property"); - for (int i = 0; i < property.getLength(); i++) //Deleting redundant properties from the document - { + for (int i = 0; i < property.getLength(); i++) { //Deleting redundant properties from the document Node prop = property.item(i); Element propElement = (Element) prop; Node propChild = propElement.getElementsByTagName("name").item(0); @@ -267,7 +259,6 @@ public class Util { rootElement.removeChild(prop); i--; } - } Iterator<Entry<String, String>> ab = map.entrySet().iterator(); @@ -289,7 +280,7 @@ public class Util { newNodeElement.appendChild(newValue); } prettyPrint(doc); - remoteFile("put", remoteFile, localFile); + remoteFile("put", remotePath, localFile); } /* @@ -340,16 +331,10 @@ public class Util { } public static void changeConfig(String remotePath) throws JSchException, SftpException { - String fileName; - remoteFile = remotePath; - - Path p = Paths.get(remoteFile); - - fileName = p.getFileName().toString(); + String fileName = Paths.get(remotePath).getFileName().toString(); backupFile = localFilePath + "backup-" + fileName; - - log.info("Copying " + backupFile + " to " + remoteFile); - remoteFile("put", remoteFile, backupFile); + log.info("Copying " + backupFile + " to " + remotePath); + remoteFile("put", remotePath, backupFile); } public static Map<String, String> mapFromXProperties(XProperties xProperties) { http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java index 60cf26c..2188363 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java @@ -465,13 +465,15 @@ public class ITListQueryTest extends BaseTestClass { Thread.sleep(1000); String startTime2 = String.valueOf(System.currentTimeMillis()); + Thread.sleep(2000); QueryHandle q2 = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_QUERY, queryName2, diffSession1).getData(); + Thread.sleep(2000); String endTime2 = String.valueOf(System.currentTimeMillis()); Thread.sleep(1000); String startTime3 = String.valueOf(System.currentTimeMillis()); - Thread.sleep(1000); + Thread.sleep(2000); QueryHandle q3 = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_QUERY, queryName3, diffSession2).getData(); qHelper.killQueryByQueryHandle(q3); String endTime3 = String.valueOf(System.currentTimeMillis()); http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java index bf40c74..7be0936 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java @@ -375,16 +375,8 @@ public class ITPreparedQueryTests extends BaseTestClass { //TODO : Destroy by user is not working, Commented the fail part, Uncomment it when fixed - String user2 = "diff", pass = "diff"; - String session1 = sHelper.openNewSession(user2, pass); - - FormBuilder formData=new FormBuilder(); - formData.add("sessionid", session1); - formData.add("type", "jar"); - formData.add("path", "file:///usr/local/lens/webapp/lens-server/WEB-INF/lib/lens-query-lib-1.2.5.jar"); - - Response response=lens.sendForm("put", "/session/resources/add", formData); - AssertUtil.assertSucceededResponse(response); + String user = "diff", pass = "diff"; + String session1 = sHelper.openNewSession(user, pass, lens.getCurrentDB()); QueryPrepareHandle queryPrepareHandle1 = qHelper.submitPreparedQuery(QueryInventory.QUERY); Assert.assertNotEquals(queryPrepareHandle1, null, "Query Execute Failed marker"); @@ -394,19 +386,19 @@ public class ITPreparedQueryTests extends BaseTestClass { Assert.assertNotEquals(queryPrepareHandle2, null, "Query Execute Failed"); logger.info("PREPARE QUERY HANDLE : " + queryPrepareHandle2); - response = qHelper.getPreparedQuery(queryPrepareHandle1); - AssertUtil.assertSucceededResponse(response); + Response response = qHelper.getPreparedQuery(queryPrepareHandle1); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); response = qHelper.getPreparedQuery(queryPrepareHandle2); - AssertUtil.assertSucceededResponse(response); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); qHelper.destroyPreparedQuery(null, lens.getUserName()); response = qHelper.getPreparedQuery(queryPrepareHandle1); AssertUtil.assertFailedResponse(response); response = qHelper.getPreparedQuery(queryPrepareHandle2); - //AssertUtil.assertSucceededResponse(response); +// Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); - qHelper.destroyPreparedQuery(null, user2); + qHelper.destroyPreparedQuery(null, user); response = qHelper.getPreparedQuery(queryPrepareHandle1); AssertUtil.assertFailedResponse(response); http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java index e16b220..f18053d 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java @@ -118,12 +118,12 @@ public class ITQueryApiTests extends BaseTestClass { Assert.assertNull(qH); } - /* Bug : LENS-1005 + /* Bug : LENS-1005 Fails bcos json output cannot be input for any API. - Here query handle is in json which caanot be passed to get query status + Here query handle is in json which cannot be passed to get query status */ - @Test + @Test(enabled = false) public void testExecuteJson() throws Exception { QueryHandle handle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY, null, sessionHandleString, null, MediaType.APPLICATION_JSON).getData(); @@ -321,8 +321,8 @@ public class ITQueryApiTests extends BaseTestClass { while (lq.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) { Response response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", servLens, null, query); - AssertUtil.assertFailedResponse(response); - System.out.println("trial : " + i++); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + logger.info("trial : " + i++); lq = qHelper.getLensQuery(sessionHandleString, queryHandle); } http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java index 8aabe2d..e588c1b 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java @@ -303,6 +303,7 @@ public class KillQueryTests extends BaseTestClass { logger.info("Start Time of 1st Query : " + startTime1); Thread.sleep(1000); QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_QUERY).getData(); + Thread.sleep(1000); String endTime1 = String.valueOf(System.currentTimeMillis()); logger.info("End Time of 1st Query : " + endTime1); @@ -312,6 +313,7 @@ public class KillQueryTests extends BaseTestClass { logger.info("Start Time of 2nd Query : " + startTime2); Thread.sleep(1000); QueryHandle queryHandle2 = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_QUERY).getData(); + Thread.sleep(1000); String endTime2 = String.valueOf(System.currentTimeMillis()); logger.info("End Time of 2nd Query : " + endTime2); @@ -347,7 +349,7 @@ public class KillQueryTests extends BaseTestClass { Thread.sleep(1000); QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_QUERY, queryName1).getData(); - logger.info("1st QUERY HANDLE : " + queryHandle1); + Thread.sleep(1000); String endTime1 = String.valueOf(System.currentTimeMillis()); logger.info("End Time of 1st Query : " + endTime1); http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java index 62dd0ba..d6a0ba9 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java @@ -143,14 +143,10 @@ public class SessionResourceTests extends BaseTestClass { //Negative Test Case @Test public void testSessionGetUndefinedParams() throws Exception { - String undefinedParamsKey = "test123"; MapBuilder query = new MapBuilder("sessionid", sessionHandleString, "key", undefinedParamsKey); Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); - AssertUtil.assertSucceededResponse(response); - StringList strList = response.readEntity(new GenericType<StringList>(StringList.class)); - HashMap<String, String> map = Util.stringListToMap(strList); - Assert.assertNull(map, "Get should have returned empty params list, but didnt"); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); } /* http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java new file mode 100644 index 0000000..f326a4f --- /dev/null +++ b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java @@ -0,0 +1,601 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.regression.config; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + + +import org.apache.lens.api.StringList; +import org.apache.lens.api.error.LensHttpStatus; +import org.apache.lens.api.query.InMemoryQueryResult; +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.regression.core.constants.MetastoreURL; +import org.apache.lens.regression.core.constants.QueryInventory; +import org.apache.lens.regression.core.constants.QueryURL; +import org.apache.lens.regression.core.constants.SessionURL; +import org.apache.lens.regression.core.helpers.LensServerHelper; +import org.apache.lens.regression.core.helpers.MetastoreHelper; +import org.apache.lens.regression.core.helpers.QueryHelper; +import org.apache.lens.regression.core.helpers.ServiceManagerHelper; +import org.apache.lens.regression.core.helpers.SessionHelper; +import org.apache.lens.regression.core.testHelper.BaseTestClass; +import org.apache.lens.regression.core.type.FormBuilder; +import org.apache.lens.regression.core.type.MapBuilder; +import org.apache.lens.regression.util.AssertUtil; +import org.apache.lens.regression.util.Util; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; + +import org.apache.log4j.Logger; + +import org.testng.Assert; +import org.testng.annotations.*; + +import com.jcraft.jsch.JSchException; + + +public class ITServerConfigTests extends BaseTestClass { + + private WebTarget servLens; + private String sessionHandleString; + + LensServerHelper lens = getLensServerHelper(); + MetastoreHelper mHelper = getMetastoreHelper(); + SessionHelper sHelper = getSessionHelper(); + QueryHelper qHelper = getQueryHelper(); + + private final String confFilePath = lens.getServerDir() + "/conf/lens-site.xml"; + private final String backupConfFilePath = lens.getServerDir() + "/conf/backup-lens-site.xml"; + private static String lensKillCmd = "ps -ef | grep -i lens | grep -v lens-client | grep -v \"grep\" | " + + "awk '{print $2}' | xargs -L1 kill -9"; + + private static Logger logger = Logger.getLogger(ITServerConfigTests.class); + + + @BeforeClass(alwaysRun = true) + public void initialize() throws IOException, JSchException { + servLens = ServiceManagerHelper.init(); + } + + @BeforeMethod(alwaysRun = true) + public void setUp(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + Util.runRemoteCommand("cp " + confFilePath + " " + backupConfFilePath); + } + + + @AfterMethod(alwaysRun = true) + public void restoreConfig() throws JSchException, IOException, LensException, InterruptedException { + logger.info("Executing after method\n"); + Util.runRemoteCommand("cp " + backupConfFilePath + " " + confFilePath); + lens.restart(); + } + + + // Session is not getting timedout, + // Opened JIRA lens-295 + + @Test(enabled = false) + public void testServerSessionTimeoutSeconds() throws Exception { + + String sessionHandle = null; + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SESSION_TIMEOUT_SECONDS, "30"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + sessionHandle = sHelper.openNewSession("user", "pass"); + sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); + + // Waiting for session timeout + Thread.sleep(40000); + + MapBuilder query = new MapBuilder("sessionid", sessionHandle, "key", LensConfConstants.SESSION_CLUSTER_USER); + Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + } finally { + if (sessionHandle!=null) { + sHelper.closeNewSession(sessionHandle); + } + } + } + + /* + * for "lens.server.restart.enabled" and "lens.server.recover.onrestart" + */ + + + @DataProvider(name = "boolean_values") + public Object[][] data() { + String[][] testData = {{"true"}, {"false"}}; + return testData; + } + + @Test(enabled = true, dataProvider = "boolean_values") + public void testServerRestartEnabled(String restartEnabled) throws Exception { + + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSISTENCE_ENABLED, + restartEnabled); + Util.changeConfig(map, confFilePath); + lens.restart(); + + String session = sHelper.openNewSession("user", "pass"); + lens.restart(); + + MapBuilder query = new MapBuilder("sessionid", session); + Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); + + if (restartEnabled.equalsIgnoreCase("true")) { + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + sHelper.closeNewSession(session); + } else { + Assert.assertEquals(response.getStatus(), Response.Status.GONE.getStatusCode()); + } + } + + /* + * Test for Property lens.server.snapshot.interval + */ + + @Test(enabled = true) + public void testSnapshotInterval() throws Exception { + + String sessionHandle = null; + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS, + "10000"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + sessionHandle = sHelper.openNewSession("user", "pass"); + sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); + //Waiting for snapshot interval time + Thread.sleep(11000); + Util.runRemoteCommand(lensKillCmd); //killing so that lens shouldn't have gracefull shutdown + lens.restart(); + + String value = sHelper.getSessionParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER); + Assert.assertEquals(value, "test"); + + } finally { + if (sessionHandle != null) { + sHelper.closeNewSession(sessionHandle); + } + } + + } + + /* + * Negative Test for Property lens.server.snapshot.interval + */ + + + @Test(enabled = true) + public void negativeTestSnapshotInterval() throws Exception { + + String sessionHandle = null; + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS, + "50000"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + sessionHandle = sHelper.openNewSession("user", "pass"); + sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); + + Util.runRemoteCommand(lensKillCmd); + lens.restart(); + + MapBuilder query = new MapBuilder("sessionid", sessionHandle, "key", LensConfConstants.SESSION_CLUSTER_USER); + Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); + Assert.assertEquals(response.getStatus(), Response.Status.GONE.getStatusCode(), "Snapshot interval test failed"); + + } finally { + if (sessionHandle != null) { + sHelper.closeNewSession(sessionHandle); + } + } + } + + + /* + * Test for Property lens.server.persist.location + */ + + @DataProvider(name = "location_provider") + public Object[][] locationProvider() { + String[][] locations = {{"file:///tmp/lensserver"}, {lens.getServerHdfsUrl() + "/tmp/lensserver"}}; + return locations; + } + + @Test(enabled = true, dataProvider = "location_provider") + public void testSessionPersistLocation(String location) throws Exception { + + String sessionHandle = null; + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSIST_LOCATION, location); + Util.changeConfig(map, confFilePath); + lens.restart(); + + sessionHandle = sHelper.openNewSession("user", "pass"); + sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); + + lens.restart(); + + String value = sHelper.getSessionParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER); + Assert.assertEquals(value, "test"); + + } finally { + if (sessionHandle != null) { + sHelper.closeNewSession(sessionHandle); + } + } + } + + + @DataProvider(name = "query_provider") + public Object[][] queryProvider() { + String[][] query = {{QueryInventory.HIVE_CUBE_QUERY}, {QueryInventory.JDBC_CUBE_QUERY}}; + return query; + } + + + /* + * Test for Property lens.server.mode=READ_ONLY + */ + + //This is failing + @Test(enabled = false) + public void testServerModeReadOnly() throws Exception { + + try { + sessionHandleString = lens.openSession(); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "READ_ONLY"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + sHelper.setAndValidateParam(sessionHandleString, LensConfConstants.SESSION_CLUSTER_USER, "test"); + + FormBuilder formData = new FormBuilder(); + formData.add("sessionid", sessionHandleString); + formData.add("query", QueryInventory.QUERY); + formData.add("operation", "EXECUTE"); + formData.add("conf", "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />"); + Response response = lens.sendForm("post", QueryURL.QUERY_URL, formData); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + response = lens.sendQuery("get", QueryURL.QUERYAPI_BASE_URL, query); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + + } finally { + lens.closeSession(); + } + } + + /* + * Test for Property lens.server.mode=METASTORE_READONLY,METASTORE_NODROP,OPEN + */ + + + @Test(enabled = true) + public void testServerMode() throws Exception { + + try { + sessionHandleString = lens.openSession(lens.getCurrentDB()); + String newDb = "TestMetastoreService_testDb1"; + + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "METASTORE_READONLY"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + Response response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + + map.put(LensConfConstants.SERVER_MODE, "METASTORE_NODROP"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); + AssertUtil.assertSucceededResponse(response); + StringList allDb = mHelper.listDatabases(); + Assert.assertTrue(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); + + query.put("cascade", "true"); + response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + + map.put(LensConfConstants.SERVER_MODE, "OPEN"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null); + AssertUtil.assertSucceededResponse(response); + allDb = mHelper.listDatabases(); + Assert.assertFalse(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); + + } finally { + if (sessionHandleString != null) { + lens.closeSession(); + } + } + } + + + /* + * Test for Lens statistics related Properties + */ + + //TODO : Need to implement this correctly + //Rollover value is being ignored + @Test(enabled = false) + public void testLensStatistics() throws Exception { + + try { + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.STATS_ROLLUP_SCAN_RATE, "60000", + LensConfConstants.STATISTICS_DATABASE_KEY, "stats", + LensConfConstants.STATISTICS_WAREHOUSE_KEY, lens.getServerHdfsUrl() + "/tmp/lens/statistics/warehouse"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + FormBuilder formData = new FormBuilder(); + formData.add("sessionid", sessionHandleString); + formData.add("type", "jar"); + formData.add("path", "file:///usr/local/lens/webapp/lens-server/WEB-INF/lib/lens-query-lib-1.2.3-SNAPSHOT.jar"); + String time = String.valueOf(System.currentTimeMillis()); + Response response = lens.sendForm("put", "/session/resources/add", formData); + AssertUtil.assertSucceededResponse(response); + + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(QueryInventory.WRONG_QUERY).getData(); + LensQuery lensQuery1 = qHelper.waitForCompletion(queryHandle1); + + Thread.sleep(120000); + mHelper.setCurrentDatabase("stats"); + + QueryHandle statsQuery = (QueryHandle) qHelper.executeQuery("select handle from stats.queryexecutionstatistics") + .getData(); + lensQuery = qHelper.waitForCompletion(statsQuery); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + InMemoryQueryResult resultSet = (InMemoryQueryResult) qHelper.getResultSet(statsQuery, "0", "100"); + for (int i = 0; i < resultSet.getRows().size(); i++) { + logger.info(resultSet.getRows().get(i).toString()); + } + Assert.assertTrue(resultSet.getRows().contains(queryHandle), "lensStats are not Saved"); + Assert.assertTrue(resultSet.getRows().contains(queryHandle1), "lensStats are not Saved"); + + } finally { + sHelper.closeSession(); + } + } + + //TODO : Add for all possible combination of enablePersistentResultSet and enablePersistentResultSetInDriver + + /* + * Test for Property lens.server.max.finished.queries for persistent result set + */ + + //This is failing + @Test(enabled = true) + public void testQueryResultRetention() throws Exception { + try { + sessionHandleString = lens.openSession(lens.getCurrentDB()); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.RESULTSET_PURGE_ENABLED, "true", + LensConfConstants.RESULTSET_PURGE_INTERVAL_IN_SECONDS, "10", + LensConfConstants.QUERY_RESULTSET_RETENTION, "20 sec", + LensConfConstants.HDFS_OUTPUT_RETENTION, "60 min"); + + Util.changeConfig(map, confFilePath); + lens.restart(); + + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + Response response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", + servLens, null, query); + AssertUtil.assertSucceededResponse(response); + + Thread.sleep(40000); + + response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", + servLens, null, query); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + } finally { + if (sessionHandleString != null) { + lens.closeSession(); + } + } + } + + + /* + * InMemoryResultSet should be purged after ttl time is over or its read once + */ + + @Test(enabled = true, dataProvider = "query_provider") + public void testInMemoryPurger(String query) throws Exception { + + sessionHandleString = lens.openSession(lens.getCurrentDB()); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_MAIL_NOTIFY, "false"); + + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS, "20", + LensConfConstants.PURGE_INTERVAL, "10000"); //in millis + Util.changeConfig(map, confFilePath); + lens.restart(); + + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(query).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + Response response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + + Thread.sleep(30000); //waiting till query gets purged ( ttl + purge interval time) + + response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + } finally { + if (sessionHandleString != null) { + sHelper.closeNewSession(sessionHandleString); + } + } + } + + // Behaviour is not the same for hive query before result is purged + @Test(enabled = true) + public void readInmemoryTwiceBeforePurgerTime() throws Exception { + + sessionHandleString = lens.openSession(lens.getCurrentDB()); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_MAIL_NOTIFY, "false"); + + try { + + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS, "500", + LensConfConstants.PURGE_INTERVAL, "10000"); + Util.changeConfig(map, confFilePath); + lens.restart(); + + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Query did not succeed"); + + Response response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + + response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + //Currently its throwing 500 which needs to be fixed. LENS-823 + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + } finally { + lens.closeSession(); + } + } + + + @Test(enabled = true) + public void testMaxSessionPerUser() throws Exception { + + String user = "test", pwd = "test"; + List<String> sessionList = new ArrayList<String>(); + int maxSession = 3; + + try { + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.MAX_SESSIONS_PER_USER, + Integer.toString(maxSession)); + Util.changeConfig(map, confFilePath); + lens.restart(); + + for (int i = 1; i <= maxSession; i++) { + sessionList.add(sHelper.openNewSession(user, pwd, lens.getCurrentDB())); + } + + Response response = sHelper.openSessionReturnResponse(user, pwd, lens.getCurrentDB(), null); + Assert.assertEquals(response.getStatus(), LensHttpStatus.TOO_MANY_REQUESTS.getStatusCode()); + + } finally { + for (String session : sessionList) { + if (session != null) { + sHelper.closeNewSession(session); + } + } + } + } + + @Test(enabled = true) + public void testMaxSessionPerUserDelete() throws Exception { + + String user1 = "test1", user2 = "test2"; + String pwd1 = "test1", pwd2 = "test2"; + + List<String> sessionList1 = new ArrayList<String>(); + List<String> sessionList2 = new ArrayList<String>(); + int maxSession = 5; + + try { + + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.MAX_SESSIONS_PER_USER, + Integer.toString(maxSession)); + Util.changeConfig(map, confFilePath); + lens.restart(); + + for (int i = 0; i < maxSession; i++) { + sessionList1.add(sHelper.openNewSession(user1, pwd1, lens.getCurrentDB())); + sessionList2.add(sHelper.openNewSession(user2, pwd2, lens.getCurrentDB())); + } + + for (int i = 0; i < maxSession; i++) { + + Response response1 = sHelper.openSessionReturnResponse(user1, pwd1, null, null); + Response response2 = sHelper.openSessionReturnResponse(user2, pwd2, null, null); + + Assert.assertEquals(response1.getStatus(), LensHttpStatus.TOO_MANY_REQUESTS.getStatusCode()); + Assert.assertEquals(response2.getStatus(), LensHttpStatus.TOO_MANY_REQUESTS.getStatusCode()); + + sHelper.closeNewSession(sessionList1.remove(1)); + sHelper.closeNewSession(sessionList2.remove(1)); + + sessionList1.add(sHelper.openNewSession(user1, pwd1, lens.getCurrentDB())); + sessionList2.add(sHelper.openNewSession(user2, pwd2, lens.getCurrentDB())); + } + + }finally { + for (int i = 0; i < sessionList1.size(); i++) { + sHelper.closeNewSession(sessionList1.get(i)); + sHelper.closeNewSession(sessionList2.get(i)); + } + } + } +} +