LENS-1216 : Add priority tests to regression
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/66b7f47f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/66b7f47f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/66b7f47f Branch: refs/heads/master Commit: 66b7f47ff14a7b0e50f0fd58ac64fbf1ddc0a19f Parents: 2de1b52 Author: Archana H <archa...@apache.org> Authored: Wed Jul 20 12:56:10 2016 +0530 Committer: Archana H <archa...@apache.org> Committed: Wed Jul 20 12:56:10 2016 +0530 ---------------------------------------------------------------------- .../regression/core/constants/MetastoreURL.java | 1 - .../core/helpers/MetastoreHelper.java | 12 +- .../core/helpers/ServiceManagerHelper.java | 103 +-------- .../regression/core/helpers/SessionHelper.java | 53 +++-- .../apache/lens/regression/util/AssertUtil.java | 2 +- .../org/apache/lens/regression/util/Util.java | 79 ++++++- .../apache/lens/regression/ITMetricsTest.java | 6 +- .../apache/lens/regression/ITPriorityTests.java | 214 ++++++++++++++++++ .../apache/lens/regression/ITRestartTests.java | 14 +- .../lens/regression/ITStreamingTests.java | 4 +- .../apache/lens/regression/SessionTests.java | 56 +++-- .../lens/regression/client/ITListQueryTest.java | 16 +- .../regression/client/ITPreparedQueryTests.java | 11 +- .../lens/regression/client/ITQueryApiTests.java | 21 +- .../lens/regression/client/KillQueryTests.java | 10 +- .../regression/client/SessionResourceTests.java | 40 ++-- .../regression/config/ITServerConfigTests.java | 219 ++++++++----------- .../lens/regression/throttling/ITCostTests.java | 114 ++++------ .../lens/regression/throttling/Throttling.java | 25 +-- 19 files changed, 595 insertions(+), 405 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java index 84cb310..43d0b5c 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java @@ -35,5 +35,4 @@ public class MetastoreURL { public static final String METASTORE_FLATTENED_URL = METASTORE_BASE_URL + "/flattened"; public static final String METASTORE_NATIVETABLES_URL = METASTORE_BASE_URL + "/nativetables"; public static final String METASTORE_STORAGES_URL = METASTORE_BASE_URL + "/storages"; - } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java index f60ecef..e993256 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java @@ -19,10 +19,10 @@ package org.apache.lens.regression.core.helpers; +import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - import javax.xml.bind.JAXBException; import org.apache.lens.api.DateTime; @@ -359,4 +359,14 @@ public class MetastoreHelper extends ServiceManagerHelper { getLatestDate(sessionHandleString); } + public StringList getAllFactsOfCube(String cubeName, String sessionHandleString) + throws InstantiationException, IllegalAccessException, JAXBException, LensException { + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + Response response = this.exec("get", MetastoreURL.METASTORE_CUBES_URL + "/" + cubeName + "/facts", servLens, null, + query, MediaType.APPLICATION_XML_TYPE); + AssertUtil.assertSucceededResponse(response); + StringList factList = response.readEntity(new GenericType<StringList>(){}); + return factList; + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java index a1d75ea..eb49827 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java @@ -29,19 +29,10 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; -import javax.xml.bind.JAXBException; - -import org.apache.lens.api.APIResult; -import org.apache.lens.api.LensConf; -import org.apache.lens.regression.core.constants.SessionURL; import org.apache.lens.regression.core.type.FormBuilder; import org.apache.lens.regression.core.type.MapBuilder; -import org.apache.lens.regression.util.AssertUtil; import org.apache.lens.regression.util.Util; -import org.apache.lens.server.api.error.LensException; -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.MultiPartFeature; @@ -61,34 +52,12 @@ public abstract class ServiceManagerHelper { private static final String JOB_CONF_URL = "job.conf.url"; private static final String START_DATE = "query.start.date"; - protected static String sessionHandleString; protected static WebTarget servLens; - - protected String baseUrl; - protected String adminUrl; - protected String userName; - protected String password; - protected String serverDir; - protected String clientDir; - protected String serverHdfsUrl; - protected String currentDB; - protected String jobConfUrl; - protected String startDate; - + protected Properties configProp = null; public ServiceManagerHelper(String envFileName) { - Properties prop = Util.getPropertiesObj(envFileName); - this.baseUrl = prop.getProperty(LENS_BASE_URL); - this.adminUrl = prop.getProperty(LENS_ADMIN_URL); - this.userName = prop.getProperty(LENS_USERNAME); - this.password = prop.getProperty(LENS_PASSWORD); - this.serverDir = prop.getProperty(LENS_SERVER_DIR); - this.clientDir = prop.getProperty(LENS_CLIENT_DIR); - this.serverHdfsUrl = prop.getProperty(LENS_SERVER_HDFS_URL); - this.currentDB = prop.getProperty(LENS_CURRENT_DB); - this.jobConfUrl = prop.getProperty(JOB_CONF_URL); - this.startDate = prop.getProperty(START_DATE); + configProp = Util.getPropertiesObj(envFileName); } public ServiceManagerHelper() { @@ -106,40 +75,36 @@ public abstract class ServiceManagerHelper { return servLens; } - public static String getSessionHandle() { - return sessionHandleString; - } - public static URI getServiceURI(String baseUri) { return UriBuilder.fromUri(baseUri).build(); } public String getBaseUrl() { - return baseUrl; + return configProp.getProperty(LENS_BASE_URL); } public String getAdminUrl() { - return adminUrl; + return configProp.getProperty(LENS_ADMIN_URL); } public String getUserName() { - return userName; + return configProp.getProperty(LENS_USERNAME); } public String getPassword() { - return password; + return configProp.getProperty(LENS_PASSWORD); } public String getServerDir() { - return serverDir; + return configProp.getProperty(LENS_SERVER_DIR); } public String getClientDir() { - return clientDir; + return configProp.getProperty(LENS_CLIENT_DIR); } public String getServerHdfsUrl() { - return serverHdfsUrl; + return configProp.getProperty(LENS_SERVER_HDFS_URL); } public String getSessionHandleString() { @@ -147,58 +112,15 @@ public abstract class ServiceManagerHelper { } public String getCurrentDB() { - return currentDB; + return configProp.getProperty(LENS_CURRENT_DB); } public String getJobConfUrl() { - return jobConfUrl; + return configProp.getProperty(JOB_CONF_URL); } public String getStartDate() { - return startDate; - } - - - public String openSession(String database) throws JAXBException, LensException { - FormBuilder formData = new FormBuilder(); - formData.add("username", this.getUserName()); - formData.add("password", this.getPassword()); - if (database != null) { - formData.add("database", database); - } - - LensConf conf = new LensConf(); - formData.getForm().bodyPart( - new FormDataBodyPart(FormDataContentDisposition.name("sessionconf").fileName("sessionconf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - formData.add("sessionconf", conf.toString(), MediaType.APPLICATION_JSON_TYPE); - - Response response = this.exec("post", SessionURL.SESSION_BASE_URL, ServiceManagerHelper.servLens, null, null, - MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm()); - AssertUtil.assertSucceededResponse(response); - sessionHandleString = response.readEntity(String.class); - log.info("Session Handle String:{}", sessionHandleString); - return sessionHandleString; - } - - public String openSession() throws JAXBException, LensException { - return openSession(null); - } - - public void closeSession() throws JAXBException, LensException { - MapBuilder query = new MapBuilder("sessionid", sessionHandleString); - Response response = this.exec("delete", SessionURL.SESSION_BASE_URL, ServiceManagerHelper.servLens, null, query); - APIResult result = response.readEntity(APIResult.class); - if (result.getStatus() != APIResult.Status.SUCCEEDED) { - throw new LensException("Status should be SUCCEEDED"); - } - if (response.getStatus() != 200) { - throw new LensException("Status code should be 200"); - } - if (result.getMessage() == null) { - throw new LensException("Status message is null"); - } - log.info("Closed Session : {}", sessionHandleString); + return configProp.getProperty(START_DATE); } public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers, @@ -238,7 +160,6 @@ public abstract class ServiceManagerHelper { } Invocation.Builder build = builder.request(outputMediaType); - functionName = "exec" + functionName.toUpperCase(); try { http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java index f7ec3e1..6c01d1e 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java @@ -27,7 +27,6 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.lens.api.LensConf; -import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.StringList; import org.apache.lens.regression.core.constants.SessionURL; import org.apache.lens.regression.core.type.FormBuilder; @@ -55,7 +54,6 @@ public class SessionHelper extends ServiceManagerHelper { /** * Open a New Session - * * @param userName * @param password * @param database @@ -76,46 +74,48 @@ public class SessionHelper extends ServiceManagerHelper { MediaType.APPLICATION_XML_TYPE)); formData.add("sessionconf", conf.toString(), MediaType.APPLICATION_JSON_TYPE); - Response response = this - .exec("post", "/session", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, outputMediaType, - formData.getForm()); + Response response = this.exec("post", SessionURL.SESSION_BASE_URL, servLens, null, null, + MediaType.MULTIPART_FORM_DATA_TYPE, outputMediaType, formData.getForm()); + return response; } - public String openNewSession(String userName, String password, String database, String outputMediaType) - throws LensException { - Response response = openSessionReturnResponse(userName, password, database, outputMediaType); + public String openSession(String database) throws LensException { + Response response = openSessionReturnResponse(this.getUserName(), this.getPassword(), database, + MediaType.APPLICATION_XML); AssertUtil.assertSucceededResponse(response); - String newSessionHandleString = response.readEntity(String.class); - log.info("Session Handle String:{}", newSessionHandleString); - return newSessionHandleString; + sessionHandleString = response.readEntity(String.class); + log.info("Session Handle String:{}", sessionHandleString); + return sessionHandleString; } - public String openNewSession(String userName, String password) throws LensException { - return openNewSession(userName, password, null, MediaType.APPLICATION_XML); + public String openSession() throws LensException { + return openSession(null); } - public String openNewSession(String userName, String password, String database) throws LensException { - return openNewSession(userName, password, database, MediaType.APPLICATION_XML); - } - - public LensSessionHandle openNewSessionJson(String userName, String password, String database, String outputMediaType) + public String openSession(String userName, String password, String database, String outputMediaType) throws LensException { - Response response = openSessionReturnResponse(userName, password, database, outputMediaType); AssertUtil.assertSucceededResponse(response); - LensSessionHandle newSessionHandleString = response.readEntity(new GenericType<LensSessionHandle>(){}); + String newSessionHandleString = response.readEntity(String.class); log.info("Session Handle String:{}", newSessionHandleString); return newSessionHandleString; } + public String openSession(String userName, String password) throws LensException { + return openSession(userName, password, null, MediaType.APPLICATION_XML); + } + + public String openSession(String userName, String password, String database) throws LensException { + return openSession(userName, password, database, MediaType.APPLICATION_XML); + } + /** * Close a Session * @param sessionHandleString */ - public void closeNewSession(String sessionHandleString, String outputMediaType) throws LensException { - + public void closeSession(String sessionHandleString, String outputMediaType) throws LensException { MapBuilder query = new MapBuilder("sessionid", sessionHandleString); Response response = this.exec("delete", SessionURL.SESSION_BASE_URL, servLens, null, query, null, outputMediaType, null); @@ -123,8 +123,12 @@ public class SessionHelper extends ServiceManagerHelper { log.info("Closed Session : {}", sessionHandleString); } - public void closeNewSession(String sessionHandleString) throws LensException { - closeNewSession(sessionHandleString, null); + public void closeSession(String sessionHandleString) throws LensException { + closeSession(sessionHandleString, null); + } + + public void closeSession() throws LensException { + closeSession(sessionHandleString, null); } /** @@ -134,6 +138,7 @@ public class SessionHelper extends ServiceManagerHelper { * @param param * @param value */ + public void setAndValidateParam(String sessionHandleString, String param, String value) throws Exception { FormBuilder formData = new FormBuilder(); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java index c4763aa..b2af76b 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java @@ -46,7 +46,7 @@ public class AssertUtil { if (response.getStatus() != 200) { throw new LensException("Status code should be 200"); } - APIResult result = Util.getApiResult(response); + APIResult result = response.readEntity(APIResult.class); if (result.getStatus() != APIResult.Status.SUCCEEDED) { throw new LensException("Status should be SUCCEEDED"); } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java index dcf5e8b..2e48e80 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.lens.regression.util; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -28,6 +29,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.ws.rs.core.Response; import javax.xml.bind.*; @@ -36,10 +39,13 @@ import javax.xml.datatype.DatatypeFactory; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.*; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; - +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import org.apache.lens.api.APIResult; import org.apache.lens.api.StringList; @@ -48,13 +54,19 @@ import org.apache.lens.api.metastore.ObjectFactory; import org.apache.lens.api.metastore.XProperties; import org.apache.lens.api.metastore.XProperty; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; + import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import com.jcraft.jsch.*; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import com.jcraft.jsch.*; import lombok.extern.slf4j.Slf4j; @@ -66,6 +78,8 @@ public class Util { private static String localFile; private static String backupFile; + private static org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(Util.class); + private Util() { } @@ -392,4 +406,65 @@ public class Util { return null; } } + + public static String getJobIdFromProgressMsg(String progressMsg){ + Pattern jobIdPattern = Pattern.compile("(job_\\d*_\\d*)"); + Matcher matcher = jobIdPattern.matcher(progressMsg); + String jobid=null; + if (matcher.find()) { + jobid = matcher.group(1); + } + return jobid; + } + + public static String getMapredJobPrority(String url, String jobId) throws IOException, IllegalAccessException, + InstantiationException, ParserConfigurationException, SAXException { + + String xmlPathExpr = "/conf/property[name='mapreduce.job.priority']/value"; + String jobConfXml = sendHttpGetRequest(url.replace("JOB_ID", jobId)); + String priority = getValueFromXml(jobConfXml, xmlPathExpr); + return priority; + } + + public static String sendHttpGetRequest(String url) throws IOException { + HttpGet request = new HttpGet(url); + request.addHeader("Accept", "application/xml"); + HttpResponse response = new DefaultHttpClient().execute(request); + BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), "UTF-8")); + StringBuffer result = new StringBuffer(); + String line = ""; + while ((line = rd.readLine()) != null) { + result.append(line); + } + return result.toString(); + } + + public static Document getDocument(String xmlContent) throws ParserConfigurationException, SAXException, IOException { + + DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xmlContent)); + Document doc = db.parse(is); + return doc; + } + + public static String getValueFromXml(String xmlContent, String xmlPathExpr) { + + String value = null; + try { + Document doc = getDocument(xmlContent); + XPath xpath = XPathFactory.newInstance().newXPath(); + value = xpath.compile(xmlPathExpr).evaluate(doc); + + } catch (ParserConfigurationException e) { + logger.info(e); + } catch (SAXException e) { + logger.info(e); + } catch (IOException e) { + logger.info(e); + } catch (XPathExpressionException e) { + logger.info(e); + } + return value; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/ITMetricsTest.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITMetricsTest.java b/lens-regression/src/test/java/org/apache/lens/regression/ITMetricsTest.java index 4f28db5..dad7b07 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/ITMetricsTest.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITMetricsTest.java @@ -78,13 +78,13 @@ public class ITMetricsTest extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session for " + method.getName()); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession(Method method) throws Exception { logger.info("Closing Session for " + method.getName()); - lens.closeSession(); + sHelper.closeSession(); } @@ -134,7 +134,7 @@ public class ITMetricsTest extends BaseTestClass { HashMap<String, Integer> oldMap = getMetricsSnapshot(); HashMap<String, Integer> newMap = null; - String sessionHandleString = sHelper.openNewSession("diff", "diff"); + String sessionHandleString = sHelper.openSession("diff", "diff"); newMap = getMetricsSnapshot(); Assert.assertEquals(newMap.get(HTTP_REQUESTS_FINISHED) - oldMap.get(HTTP_REQUESTS_FINISHED), 1); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/ITPriorityTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITPriorityTests.java b/lens-regression/src/test/java/org/apache/lens/regression/ITPriorityTests.java new file mode 100644 index 0000000..1bae767 --- /dev/null +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITPriorityTests.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.lens.regression; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import javax.ws.rs.client.WebTarget; + +import org.apache.lens.api.LensConf; +import org.apache.lens.api.Priority; +import org.apache.lens.api.query.*; +import org.apache.lens.cube.parse.CubeQueryConfUtil; +import org.apache.lens.driver.hive.HiveDriver; +import org.apache.lens.regression.core.constants.DriverConfig; +import org.apache.lens.regression.core.constants.QueryInventory; +import org.apache.lens.regression.core.helpers.*; +import org.apache.lens.regression.core.testHelper.BaseTestClass; +import org.apache.lens.regression.util.Util; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; + +import org.apache.log4j.Logger; + +import org.testng.Assert; +import org.testng.annotations.*; + +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.SftpException; + +public class ITPriorityTests extends BaseTestClass{ + + WebTarget servLens; + String sessionHandleString; + + LensServerHelper lens = getLensServerHelper(); + MetastoreHelper mHelper = getMetastoreHelper(); + SessionHelper sHelper = getSessionHelper(); + QueryHelper qHelper = getQueryHelper(); + + String hiveDriver = "hive/hive1"; + String hiveDriverConfPath = lens.getServerDir() + "/conf/drivers/" + hiveDriver + "/hivedriver-site.xml"; + String lensSiteConfPath = lens.getServerDir() + "/conf/lens-site.xml"; + private String jobUrl = lens.getJobConfUrl(); + + public static final String COST_95 = QueryInventory.getQueryFromInventory("HIVE.COST_95"); + public static final String COST_60 = QueryInventory.getQueryFromInventory("HIVE.COST_60"); + public static final String COST_20 = QueryInventory.getQueryFromInventory("HIVE.COST_20"); + public static final String COST_5 = QueryInventory.getQueryFromInventory("HIVE.COST_5"); + + private static Logger logger = Logger.getLogger(ITPriorityTests.class); + + + @BeforeClass(alwaysRun = true) + public void initialize() throws Exception { + servLens = ServiceManagerHelper.init(); + HashMap<String, String> map = LensUtil.getHashMap(HiveDriver.HS2_PRIORITY_RANGES, + "HIGH,7,NORMAL,30,LOW,90,VERY_LOW"); + Util.changeConfig(map, hiveDriverConfPath); + lens.restart(); + + sessionHandleString=sHelper.openSession(lens.getCurrentDB()); + sHelper.setAndValidateParam(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + } + + @BeforeMethod(alwaysRun = true) + public void setUp(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + } + + @AfterMethod(alwaysRun=true) + public void restoreConfig() throws JSchException, IOException, LensException{ + } + + @AfterClass(alwaysRun = true) + public void cleanup() throws IOException, JSchException, LensException, InterruptedException, SftpException { + logger.info("Closing Session"); + sHelper.closeSession(); + Util.changeConfig(hiveDriverConfPath); + lens.restart(); + } + + + @DataProvider(name = "priority_check") + public Object[][] priorityCheck() { + Object[][] testData = {{COST_5, Priority.HIGH}, {COST_20, Priority.NORMAL}, {COST_60, Priority.LOW}, + {COST_95, Priority.VERY_LOW}, }; + return testData; + } + + @Test(enabled=true, dataProvider = "priority_check") + public void testPriority(String query, Priority priority) throws Exception { + QueryHandle qh = (QueryHandle) qHelper.executeQuery(query).getData(); + qHelper.waitForQueryToRun(qh); + LensQuery lq = qHelper.getLensQuery(sessionHandleString, qh); + String progressMsg = qHelper.getQueryStatus(qh).getProgressMessage(); + logger.info("Progress msg : " + progressMsg); + String jobId = Util.getJobIdFromProgressMsg(progressMsg); + Assert.assertEquals(lq.getPriority(), priority); + Assert.assertEquals(Util.getMapredJobPrority(jobUrl, jobId), priority.toString()); + } + + + @Test(enabled = true) + public void testPriorityAfterQueryPurge() throws Exception { + + try{ + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.PURGE_INTERVAL, "5000"); + Util.changeConfig(map, lensSiteConfPath); + lens.restart(); + + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.HIVE_CUBE_QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Thread.sleep(6000); + lensQuery = qHelper.getLensQuery(sessionHandleString, queryHandle); + Assert.assertEquals(lensQuery.getPriority(), Priority.HIGH); + + }catch(Exception e){ + Util.changeConfig(lensSiteConfPath); + lens.restart(); + } + } + + /* + When query executed with TIMEOUT is in running state then all queued queries will fail with + null pointer execption when getting prioity. This is becuasue there was a bug in queryWithTimeout + execution flow. It was not going in the normal execution path where priority was set + */ + + @Test(enabled = true) + public void testPriorityForTimeoutQuery() throws Exception { + + String cost60 = QueryInventory.getQueryFromInventory("HIVE.COST_60"); + List<QueryHandle> list = new ArrayList<QueryHandle>(); + + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + + QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout(cost60, "10000") + .getData(); + + for(int i = 0; i < 6; i++){ + list.add((QueryHandle) qHelper.executeQuery(QueryInventory.HIVE_CUBE_QUERY).getData()); + } + + LensQuery lq = qHelper.waitForCompletion(qhr.getQueryHandle()); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + + for(QueryHandle q : list){ + LensQuery lensQuery = qHelper.waitForCompletion(q); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + } + + + /* + LENS-1016 : When query is executed with "mapreduce.job.priority" set by user, then all queued queries will fail with + null pointer execption when getting prioity. This is becuasue priority is not set for that query. + */ + + @Test(enabled = true) + public void testExplicitPrioritySettingByUser() throws Exception { + + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "4", + DriverConfig.PRIORITY_MAX_CONCURRENT, "LOW=2,VERY_LOW=1,HIGH=4"); + String sleepQuery = QueryInventory.getSleepQuery("10"); + + try { + Util.changeConfig(map, hiveDriverConfPath); + lens.restart(); + + LensConf lensConf = new LensConf(); + lensConf.addProperty("mapreduce.job.priority", "HIGH"); + + handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, null, sessionHandleString, + lensConf).getData()); + + for(int i=0; i<6; i++){ + handleList.add((QueryHandle) qHelper.executeQuery(QueryInventory.HIVE_CUBE_QUERY).getData()); + } + + for(int i=0; i<handleList.size(); i++){ + LensQuery lq = qHelper.waitForCompletion(handleList.get(i)); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + } finally { + Util.changeConfig(hiveDriverConfPath); + lens.restart(); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/ITRestartTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITRestartTests.java b/lens-regression/src/test/java/org/apache/lens/regression/ITRestartTests.java index 203996b..70d1289 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/ITRestartTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITRestartTests.java @@ -80,14 +80,14 @@ public class ITRestartTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); if (sessionHandleString!=null) { - lens.closeSession(); + sHelper.closeSession(); } } @@ -109,7 +109,7 @@ public class ITRestartTests extends BaseTestClass { LensConf lensConf = new LensConf(); lensConf.addProperties(confMap); - String session1 = sHelper.openNewSession("diff1", "diff1", lens.getCurrentDB()); + String session1 = sHelper.openSession("diff1", "diff1", lens.getCurrentDB()); sHelper.setAndValidateParam(confMap, session1); List<QueryHandle> qList = new ArrayList<QueryHandle>(); @@ -117,12 +117,12 @@ public class ITRestartTests extends BaseTestClass { qList.add((QueryHandle)qHelper.executeQuery(query, null, null, session1, lensConf).getData()); } - String session2 = sHelper.openNewSession("diff2", "diff2", lens.getCurrentDB()); - String session3 = sHelper.openNewSession("diff3", "diff3", lens.getCurrentDB()); + String session2 = sHelper.openSession("diff2", "diff2", lens.getCurrentDB()); + String session3 = sHelper.openSession("diff3", "diff3", lens.getCurrentDB()); qHelper.waitForCompletion(qList.get(0)); Thread.sleep(5000); // wait till query gets persisted - sHelper.closeNewSession(session1); // required to hit that flow + sHelper.closeSession(session1); // required to hit that flow lens.restart(); Assert.assertTrue(Boolean.parseBoolean(sHelper.getSessionParam(session2, @@ -134,7 +134,7 @@ public class ITRestartTests extends BaseTestClass { for(int i=4; i<8; i++){ String user = "diff" + Integer.toString(i); - String session = sHelper.openNewSession(user, user, lens.getCurrentDB()); + String session = sHelper.openSession(user, user, lens.getCurrentDB()); String isDriverPersist = sHelper.getSessionParam(session, LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER); String isMailNotify = sHelper.getSessionParam(session, mailNotify); logger.info(user + " session : " + isDriverPersist); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java index d10f011..d28b2ac 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java @@ -67,13 +67,13 @@ public class ITStreamingTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @DataProvider(name="query_provider") http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java b/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java index 8b53897..255f5be 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/SessionTests.java @@ -73,13 +73,13 @@ public class SessionTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @@ -102,48 +102,58 @@ public class SessionTests extends BaseTestClass { @Test(enabled = true) public void testRunningQueryContinuationOnSessionClose() throws Exception { - String session = sHelper.openNewSession("test", "test", lens.getCurrentDB()); - List<QueryHandle> handleList = new ArrayList<QueryHandle>(); - String sleepQuery = QueryInventory.getSleepQuery("5"); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "10"); + String hiveDriverConf = lens.getServerDir() + "/conf/drivers/hive/hive1/hivedriver-site.xml"; - for(int i=1; i<=5; i++){ - handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, session).getData()); - } + try { + Util.changeConfig(map, hiveDriverConf); + lens.restart(); + + String session = sHelper.openSession("test", "test", lens.getCurrentDB()); + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + String sleepQuery = QueryInventory.getSleepQuery("5"); - Thread.sleep(3000); + for(int i=1; i<=5; i++){ + handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, session).getData()); + } + qHelper.waitForQueryToRun(handleList.get(3)); - List<QueryHandle> running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString, null, null); - sHelper.closeNewSession(session); - Assert.assertTrue(running.size() > 0); - logger.info("Running query count " + running.size()); + List<QueryHandle> running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString); + sHelper.closeSession(session); + Assert.assertTrue(running.size() > 0); + logger.info("Running query count " + running.size()); - for(QueryHandle handle : running){ - LensQuery lq = qHelper.waitForCompletion(handle); - Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + for(QueryHandle handle : running){ + LensQuery lq = qHelper.waitForCompletion(handle); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + } finally { + Util.changeConfig(hiveDriverConf); + lens.restart(); } } // Fails. Bug : LENS-904 // Check for query continuation on session close. - @Test(enabled = false) + @Test(enabled = true) public void testQueryContinuationOnSessionClose() throws Exception { HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1"); - String expHiveDriverConf = lens.getServerDir() + "/conf/drivers/hive/hive1/hivedriver-site.xml"; + String hiveDriverConf = lens.getServerDir() + "/conf/drivers/hive/hive1/hivedriver-site.xml"; try { - Util.changeConfig(map, expHiveDriverConf); + Util.changeConfig(map, hiveDriverConf); lens.restart(); - String session = sHelper.openNewSession("test", "test", lens.getCurrentDB()); + String session = sHelper.openSession("test", "test", lens.getCurrentDB()); List<QueryHandle> handleList = new ArrayList<QueryHandle>(); - String sleepQuery = QueryInventory.getSleepQuery("5"); + String sleepQuery = QueryInventory.getSleepQuery("3"); for (int i = 1; i <= 5; i++) { handleList.add((QueryHandle) qHelper.executeQuery(sleepQuery, null, session).getData()); } - sHelper.closeNewSession(session); + sHelper.closeSession(session); for (QueryHandle handle : handleList) { LensQuery lq = qHelper.waitForCompletion(handle); @@ -151,7 +161,7 @@ public class SessionTests extends BaseTestClass { } } finally { - Util.changeConfig(expHiveDriverConf); + Util.changeConfig(hiveDriverConf); lens.restart(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java index f64d966..1652ba3 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java @@ -57,7 +57,7 @@ public class ITListQueryTest extends BaseTestClass { public void initialize() throws IOException, JAXBException, LensException { servLens = ServiceManagerHelper.init(); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @BeforeMethod(alwaysRun = true) @@ -75,7 +75,7 @@ public class ITListQueryTest extends BaseTestClass { @AfterClass(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @DataProvider(name = "query-provider") @@ -193,7 +193,7 @@ public class ITListQueryTest extends BaseTestClass { public void listQuerySpecificUserAllUser() throws Exception { String diffUser = "diff"; String diffPass = "diff"; - String diffSessionHandleString = sHelper.openNewSession(diffUser, diffPass, lens.getCurrentDB()); + String diffSessionHandleString = sHelper.openSession(diffUser, diffPass, lens.getCurrentDB()); //Running Query with user1 QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(QueryInventory.QUERY).getData(); @@ -376,7 +376,7 @@ public class ITListQueryTest extends BaseTestClass { QueryHandle q2 = (QueryHandle) qHelper.executeQuery(QueryInventory.HIVE_DIM_QUERY).getData(); //Running Query with diff user - String diffSession = sHelper.openNewSession("diff", "diff", lens.getCurrentDB()); + String diffSession = sHelper.openSession("diff", "diff", lens.getCurrentDB()); QueryHandle q3 = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY, null, diffSession).getData(); QueryHandle q4 = (QueryHandle) qHelper.executeQuery(QueryInventory.HIVE_CUBE_QUERY, null, diffSession).getData(); @@ -403,7 +403,7 @@ public class ITListQueryTest extends BaseTestClass { public void listQueryByDriverNStatus() throws Exception { String user = "new", pwd = "new"; - String newSession = sHelper.openNewSession(user, pwd, mHelper.getCurrentDB()); + String newSession = sHelper.openSession(user, pwd, mHelper.getCurrentDB()); //Cancelled hive Query QueryHandle q1 = (QueryHandle) qHelper.executeQuery(sleepQuery, null, newSession).getData(); @@ -416,7 +416,7 @@ public class ITListQueryTest extends BaseTestClass { //Successful jdbc Query QueryHandle q3 = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY, null, newSession).getData(); - qHelper.waitForCompletion(newSession, q2); + qHelper.waitForCompletion(newSession, q3); //Running hive Query QueryHandle q4 = (QueryHandle) qHelper.executeQuery(sleepQuery, null, newSession).getData(); @@ -454,8 +454,8 @@ public class ITListQueryTest extends BaseTestClass { String user1 = "diff", pwd1 = "diff"; String user2 = "diff1", pwd2 = "diff1"; - String diffSession1 = sHelper.openNewSession(user1, pwd1, lens.getCurrentDB()); - String diffSession2 = sHelper.openNewSession(user2, pwd2, lens.getCurrentDB()); + String diffSession1 = sHelper.openSession(user1, pwd1, lens.getCurrentDB()); + String diffSession2 = sHelper.openSession(user2, pwd2, lens.getCurrentDB()); String startTime1 = String.valueOf(System.currentTimeMillis()); QueryHandle q1 = (QueryHandle) qHelper.executeQuery(QueryInventory.QUERY, queryName1).getData(); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java index 7be0936..6292867 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITPreparedQueryTests.java @@ -79,8 +79,7 @@ public class ITPreparedQueryTests extends BaseTestClass { public void initialize() throws IOException, JAXBException, LensException { servLens = ServiceManagerHelper.init(); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(); - mHelper.setCurrentDatabase(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @BeforeMethod(alwaysRun = true) @@ -91,7 +90,7 @@ public class ITPreparedQueryTests extends BaseTestClass { @AfterClass(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @Test @@ -240,7 +239,7 @@ public class ITPreparedQueryTests extends BaseTestClass { String user = "diff", pass = "diff"; - String session1 = sHelper.openNewSession(user, pass, lens.getCurrentDB()); + String session1 = sHelper.openSession(user, pass, lens.getCurrentDB()); QueryPrepareHandle queryPrepareHandle1 = qHelper.submitPreparedQuery(QueryInventory.QUERY); Assert.assertNotEquals(queryPrepareHandle1, null, "Query Execute Failed"); @@ -376,7 +375,7 @@ public class ITPreparedQueryTests extends BaseTestClass { //TODO : Destroy by user is not working, Commented the fail part, Uncomment it when fixed String user = "diff", pass = "diff"; - String session1 = sHelper.openNewSession(user, pass, lens.getCurrentDB()); + String session1 = sHelper.openSession(user, pass, lens.getCurrentDB()); QueryPrepareHandle queryPrepareHandle1 = qHelper.submitPreparedQuery(QueryInventory.QUERY); Assert.assertNotEquals(queryPrepareHandle1, null, "Query Execute Failed marker"); @@ -473,7 +472,7 @@ public class ITPreparedQueryTests extends BaseTestClass { String queryName1 = "queryfirst", queryName2 = "querysecond"; String user2 = "user2", pass2 = "pass2"; - String session1 = sHelper.openNewSession(user2, pass2, lens.getCurrentDB()); + String session1 = sHelper.openSession(user2, pass2, lens.getCurrentDB()); String startTime1=String.valueOf(System.currentTimeMillis()); logger.info("Start Time of first query- "+startTime1); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java index ea8d7f2..2d0c3f1 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/ITQueryApiTests.java @@ -75,13 +75,13 @@ public class ITQueryApiTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @DataProvider(name = "persistance_values") @@ -392,14 +392,13 @@ public class ITQueryApiTests extends BaseTestClass { and earlier is still queued or running, then return the same handle. */ - @DataProvider(name = "query_names") public Object[][] queryName() { String[][] testData = {{"query-name"}, {null}}; return testData; } - @Test(dataProvider = "query_names") + @Test(dataProvider = "query_names", enabled = false) public void testRunningSameNameSessionQuery(String queryName) throws Exception { String query = QueryInventory.getSleepQuery("10"); @@ -423,7 +422,7 @@ public class ITQueryApiTests extends BaseTestClass { Assert.assertEquals(resultList.get(2).getPersistedURI(), resultList.get(0).getPersistedURI()); } - @Test() + @Test(enabled = false) public void testQueuedSameNameSessionQuery() throws Exception { String query = QueryInventory.getSleepQuery("10"); @@ -460,7 +459,7 @@ public class ITQueryApiTests extends BaseTestClass { } } - @Test + @Test(enabled = false) public void differentQuerySameNameSession() throws Exception { String cost5 = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_5"), "5"); @@ -472,18 +471,18 @@ public class ITQueryApiTests extends BaseTestClass { Assert.assertFalse(handle1.getHandleIdString().equals(handle2.getHandleIdString())); } - @Test + @Test(enabled = false) public void differentSessionSameNameQuery() throws Exception { String query = QueryInventory.getSleepQuery("10"); - String session1 = sHelper.openNewSession("user1", "pwd1", lens.getCurrentDB()); - String session2 = sHelper.openNewSession("user2", "pwd2", lens.getCurrentDB()); + String session1 = sHelper.openSession("user1", "pwd1", lens.getCurrentDB()); + String session2 = sHelper.openSession("user2", "pwd2", lens.getCurrentDB()); QueryHandle handle1 = (QueryHandle) qHelper.executeQuery(query, "name", session1).getData(); QueryHandle handle2 = (QueryHandle) qHelper.executeQuery(query, "name", session2).getData(); Assert.assertFalse(handle1.getHandleIdString().equals(handle2.getHandleIdString())); } - @Test + @Test(enabled = false) public void differentNameSameSessionQuery() throws Exception { String query = QueryInventory.getSleepQuery("3"); QueryHandle handle1 = (QueryHandle) qHelper.executeQuery(query, "name1").getData(); @@ -491,7 +490,7 @@ public class ITQueryApiTests extends BaseTestClass { Assert.assertFalse(handle1.getHandleIdString().equals(handle2.getHandleIdString())); } - @Test + @Test(enabled = false) public void differentConfSameNameSessionQuery() throws Exception { String query = QueryInventory.getSleepQuery("5"); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java index cfeae1c..e7f40e7 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java @@ -62,10 +62,10 @@ public class KillQueryTests extends BaseTestClass { InstantiationException { servLens = ServiceManagerHelper.init(); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); //TODO : Enable when udf registration per driver is fixed -/* HadoopUtil.uploadJars(localJarPath + "/" + hiveUdfJar, hdfsJarPath); +/* HadoopUtil.uploadJars(localJarPath + "/" + hiveUdfJar, hdfsJarPath); logger.info("Adding jar for making query to run for longer period of time"); sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar); QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.SLEEP_FUNCTION).getData();*/ @@ -80,7 +80,7 @@ public class KillQueryTests extends BaseTestClass { @AfterClass(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @@ -106,7 +106,7 @@ public class KillQueryTests extends BaseTestClass { String diffUser = "diff"; String diffPass = "diff"; - String newSessionHandleSring = sHelper.openNewSession(diffUser, diffPass, lens.getCurrentDB()); + String newSessionHandleSring = sHelper.openSession(diffUser, diffPass, lens.getCurrentDB()); logger.info("Adding jar for making query to run for longer period of time"); sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar, newSessionHandleSring); @@ -153,7 +153,7 @@ public class KillQueryTests extends BaseTestClass { String diffUser = "diff"; String diffPass = "diff"; - String newSessionHandleSring = sHelper.openNewSession(diffUser, diffPass, lens.getCurrentDB()); + String newSessionHandleSring = sHelper.openSession(diffUser, diffPass, lens.getCurrentDB()); sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar, newSessionHandleSring); QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(sleepQuery).getData(); http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java b/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java index d6a0ba9..00fc729 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/client/SessionResourceTests.java @@ -91,13 +91,13 @@ public class SessionResourceTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } private boolean checkSessionParamMap(String sessionHandle) throws Exception { @@ -115,7 +115,7 @@ public class SessionResourceTests extends BaseTestClass { @Test public void testSessionGet() throws Exception { - String newSessionHandle = sHelper.openNewSession("diff", "diff"); + String newSessionHandle = sHelper.openSession("diff", "diff"); Assert.assertNotNull(newSessionHandle); } @@ -233,8 +233,8 @@ public class SessionResourceTests extends BaseTestClass { @Test public void testSessionGone() throws Exception { - String newSession = sHelper.openNewSession("test", "test"); - sHelper.closeNewSession(newSession); + String newSession = sHelper.openSession("test", "test"); + sHelper.closeSession(newSession); MapBuilder query = new MapBuilder("sessionid", newSession); @@ -275,20 +275,20 @@ public class SessionResourceTests extends BaseTestClass { String newDb = "opensessionwithdb"; mHelper.createDatabase(newDb); - String newSession = sHelper.openNewSession("test", "test", newDb); + String newSession = sHelper.openSession("test", "test", newDb); String curDB = mHelper.getCurrentDatabase(newSession); Assert.assertEquals(curDB, newDb, "Could not open session with passed db"); - sHelper.closeNewSession(newSession); + sHelper.closeSession(newSession); mHelper.dropDatabase(newDb); } @Test public void testOpenSessionDefault() throws Exception { - String newSession = sHelper.openNewSession("test", "test"); + String newSession = sHelper.openSession("test", "test"); String curDB = mHelper.getCurrentDatabase(newSession); Assert.assertEquals(curDB, "default", "Could not open session with passed db"); - sHelper.closeNewSession(newSession); + sHelper.closeSession(newSession); } @@ -307,7 +307,7 @@ public class SessionResourceTests extends BaseTestClass { mHelper.createDatabase(newDb); mHelper.createDatabase(newDb1); - String newSession = sHelper.openNewSession("test", "test", newDb); + String newSession = sHelper.openSession("test", "test", newDb); String curDB = mHelper.getCurrentDatabase(newSession); Assert.assertEquals(curDB, newDb, "Could not open session with passed db"); @@ -315,7 +315,7 @@ public class SessionResourceTests extends BaseTestClass { curDB = mHelper.getCurrentDatabase(newSession); Assert.assertEquals(curDB, newDb1, "Could not open session with passed db"); - sHelper.closeNewSession(newSession); + sHelper.closeSession(newSession); mHelper.dropDatabase(newDb); mHelper.dropDatabase(newDb1); } @@ -324,17 +324,21 @@ public class SessionResourceTests extends BaseTestClass { @Test(enabled = false) public void testGetSessionJson() throws Exception { - String newSessionHandle = sHelper.openNewSession("diff", "diff", null, MediaType.APPLICATION_JSON); + String newSessionHandle = sHelper.openSession("diff", "diff", null, MediaType.APPLICATION_JSON); Assert.assertNotNull(newSessionHandle); Assert.assertFalse(newSessionHandle.isEmpty()); - sHelper.closeNewSession(newSessionHandle, MediaType.APPLICATION_JSON); + sHelper.closeSession(newSessionHandle, MediaType.APPLICATION_JSON); } @Test(enabled = true) - public void testCloseSessionJson() throws Exception { - String s2 = sHelper.openNewSession("diff", "diff", null, MediaType.APPLICATION_XML); - Assert.assertNotNull(s2); - Assert.assertFalse(s2.isEmpty()); - sHelper.closeNewSession(s2, MediaType.APPLICATION_JSON); + public void assertSucceededResponse() throws Exception { + String session = sHelper.openSession("diff", "diff", null, MediaType.APPLICATION_XML); + Assert.assertNotNull(session); + Assert.assertFalse(session.isEmpty()); + + MapBuilder query = new MapBuilder("sessionid", session); + Response response = lens.exec("delete", SessionURL.SESSION_BASE_URL, servLens, null, query, null, + MediaType.APPLICATION_JSON, null); + AssertUtil.assertSucceededResponse(response); } } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java index 48ea8ec..8294075 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java @@ -89,12 +89,14 @@ public class ITServerConfigTests extends BaseTestClass { public void setUp(Method method) throws Exception { logger.info("Test Name: " + method.getName()); Util.runRemoteCommand("cp " + confFilePath + " " + backupConfFilePath); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); } @AfterMethod(alwaysRun = true) public void restoreConfig() throws JSchException, IOException, LensException, InterruptedException { logger.info("Executing after method\n"); + sHelper.closeSession(); Util.runRemoteCommand("cp " + backupConfFilePath + " " + confFilePath); lens.restart(); } @@ -112,7 +114,7 @@ public class ITServerConfigTests extends BaseTestClass { Util.changeConfig(map, confFilePath); lens.restart(); - sessionHandle = sHelper.openNewSession("user", "pass"); + sessionHandle = sHelper.openSession("user", "pass"); sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); // Waiting for session timeout @@ -124,7 +126,7 @@ public class ITServerConfigTests extends BaseTestClass { } finally { if (sessionHandle!=null) { - sHelper.closeNewSession(sessionHandle); + sHelper.closeSession(sessionHandle); } } } @@ -148,7 +150,7 @@ public class ITServerConfigTests extends BaseTestClass { Util.changeConfig(map, confFilePath); lens.restart(); - String session = sHelper.openNewSession("user", "pass"); + String session = sHelper.openSession("user", "pass"); lens.restart(); MapBuilder query = new MapBuilder("sessionid", session); @@ -156,7 +158,7 @@ public class ITServerConfigTests extends BaseTestClass { if (restartEnabled.equalsIgnoreCase("true")) { Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); - sHelper.closeNewSession(session); + sHelper.closeSession(session); } else { Assert.assertEquals(response.getStatus(), Response.Status.GONE.getStatusCode()); } @@ -176,7 +178,7 @@ public class ITServerConfigTests extends BaseTestClass { Util.changeConfig(map, confFilePath); lens.restart(); - sessionHandle = sHelper.openNewSession("user", "pass"); + sessionHandle = sHelper.openSession("user", "pass"); sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); //Waiting for snapshot interval time Thread.sleep(11000); @@ -188,7 +190,7 @@ public class ITServerConfigTests extends BaseTestClass { } finally { if (sessionHandle != null) { - sHelper.closeNewSession(sessionHandle); + sHelper.closeSession(sessionHandle); } } @@ -207,7 +209,7 @@ public class ITServerConfigTests extends BaseTestClass { Util.changeConfig(map, confFilePath); lens.restart(); - String sessionHandle = sHelper.openNewSession("user", "pass"); + String sessionHandle = sHelper.openSession("user", "pass"); sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); //killing so that lens is not stopped gracefully. @@ -239,7 +241,7 @@ public class ITServerConfigTests extends BaseTestClass { Util.changeConfig(map, confFilePath); lens.restart(); - sessionHandle = sHelper.openNewSession("user", "pass"); + sessionHandle = sHelper.openSession("user", "pass"); sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); lens.restart(); @@ -249,7 +251,7 @@ public class ITServerConfigTests extends BaseTestClass { } finally { if (sessionHandle != null) { - sHelper.closeNewSession(sessionHandle); + sHelper.closeSession(sessionHandle); } } } @@ -270,29 +272,24 @@ public class ITServerConfigTests extends BaseTestClass { @Test(enabled = false) public void testServerModeReadOnly() throws Exception { - try { - sessionHandleString = lens.openSession(); - HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "READ_ONLY"); - Util.changeConfig(map, confFilePath); - lens.restart(); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "READ_ONLY"); + Util.changeConfig(map, confFilePath); + lens.restart(); - sHelper.setAndValidateParam(sessionHandleString, LensConfConstants.SESSION_CLUSTER_USER, "test"); + sHelper.setAndValidateParam(sessionHandleString, LensConfConstants.SESSION_CLUSTER_USER, "test"); - FormBuilder formData = new FormBuilder(); - formData.add("sessionid", sessionHandleString); - formData.add("query", QueryInventory.QUERY); - formData.add("operation", "EXECUTE"); - formData.add("conf", "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />"); - Response response = lens.sendForm("post", QueryURL.QUERY_URL, formData); - Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); - - MapBuilder query = new MapBuilder("sessionid", sessionHandleString); - response = lens.sendQuery("get", QueryURL.QUERYAPI_BASE_URL, query); - Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + FormBuilder formData = new FormBuilder(); + formData.add("sessionid", sessionHandleString); + formData.add("query", QueryInventory.QUERY); + formData.add("operation", "EXECUTE"); + formData.add("conf", "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />"); + Response response = lens.sendForm("post", QueryURL.QUERY_URL, formData); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + response = lens.sendQuery("get", QueryURL.QUERYAPI_BASE_URL, query); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); - } finally { - lens.closeSession(); - } } /* @@ -303,49 +300,41 @@ public class ITServerConfigTests extends BaseTestClass { @Test(enabled = true) public void testServerMode() throws Exception { - try { - sessionHandleString = lens.openSession(lens.getCurrentDB()); - String newDb = "TestMetastoreService_testDb1"; - - HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "METASTORE_READONLY"); - Util.changeConfig(map, confFilePath); - lens.restart(); + String newDb = "TestMetastoreService_testDb1"; - MapBuilder query = new MapBuilder("sessionid", sessionHandleString); - Response response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, - null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); - Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_MODE, "METASTORE_READONLY"); + Util.changeConfig(map, confFilePath); + lens.restart(); - map.put(LensConfConstants.SERVER_MODE, "METASTORE_NODROP"); - Util.changeConfig(map, confFilePath); - lens.restart(); + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + Response response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); - response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, - null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); - AssertUtil.assertSucceededResponse(response); - StringList allDb = mHelper.listDatabases(); - Assert.assertTrue(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); + map.put(LensConfConstants.SERVER_MODE, "METASTORE_NODROP"); + Util.changeConfig(map, confFilePath); + lens.restart(); - query.put("cascade", "true"); - response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, - null, query, MediaType.APPLICATION_XML_TYPE, null); - Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); + response = mHelper.exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null, newDb); + AssertUtil.assertSucceededResponse(response); + StringList allDb = mHelper.listDatabases(); + Assert.assertTrue(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); - map.put(LensConfConstants.SERVER_MODE, "OPEN"); - Util.changeConfig(map, confFilePath); - lens.restart(); + query.put("cascade", "true"); + response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null); + Assert.assertEquals(response.getStatus(), Response.Status.METHOD_NOT_ALLOWED.getStatusCode()); - response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, - null, query, MediaType.APPLICATION_XML_TYPE, null); - AssertUtil.assertSucceededResponse(response); - allDb = mHelper.listDatabases(); - Assert.assertFalse(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); + map.put(LensConfConstants.SERVER_MODE, "OPEN"); + Util.changeConfig(map, confFilePath); + lens.restart(); - } finally { - if (sessionHandleString != null) { - lens.closeSession(); - } - } + response = mHelper.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + newDb, servLens, + null, query, MediaType.APPLICATION_XML_TYPE, null); + AssertUtil.assertSucceededResponse(response); + allDb = mHelper.listDatabases(); + Assert.assertFalse(allDb.getElements().contains(newDb.toLowerCase()), "Unable to Create DB"); } @@ -359,7 +348,6 @@ public class ITServerConfigTests extends BaseTestClass { public void testLensStatistics() throws Exception { try { - sessionHandleString = sHelper.openSession(lens.getCurrentDB()); HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.STATS_ROLLUP_SCAN_RATE, "60000", LensConfConstants.STATISTICS_DATABASE_KEY, "stats", LensConfConstants.STATISTICS_WAREHOUSE_KEY, lens.getServerHdfsUrl() + "/tmp/lens/statistics/warehouse"); @@ -410,40 +398,33 @@ public class ITServerConfigTests extends BaseTestClass { //This is failing @Test(enabled = true) public void testQueryResultRetention() throws Exception { - try { - sessionHandleString = lens.openSession(lens.getCurrentDB()); - sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); - sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.RESULTSET_PURGE_ENABLED, "true", - LensConfConstants.RESULTSET_PURGE_INTERVAL_IN_SECONDS, "10", - LensConfConstants.QUERY_RESULTSET_RETENTION, "20 sec", - LensConfConstants.HDFS_OUTPUT_RETENTION, "60 min"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - Util.changeConfig(map, confFilePath); - lens.restart(); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.RESULTSET_PURGE_ENABLED, "true", + LensConfConstants.RESULTSET_PURGE_INTERVAL_IN_SECONDS, "10", + LensConfConstants.QUERY_RESULTSET_RETENTION, "20 sec", + LensConfConstants.HDFS_OUTPUT_RETENTION, "60 min"); - MapBuilder query = new MapBuilder("sessionid", sessionHandleString); + Util.changeConfig(map, confFilePath); + lens.restart(); - QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); - LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); - Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + MapBuilder query = new MapBuilder("sessionid", sessionHandleString); - Response response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", - servLens, null, query); - AssertUtil.assertSucceededResponse(response); + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - Thread.sleep(40000); + Response response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", + servLens, null, query); + AssertUtil.assertSucceededResponse(response); - response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", - servLens, null, query); - Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + Thread.sleep(40000); - } finally { - if (sessionHandleString != null) { - lens.closeSession(); - } - } + response = qHelper.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", + servLens, null, query); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); } @@ -454,7 +435,6 @@ public class ITServerConfigTests extends BaseTestClass { @Test(enabled = true, dataProvider = "query_provider") public void testInMemoryPurger(String query) throws Exception { - sessionHandleString = lens.openSession(lens.getCurrentDB()); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); sHelper.setAndValidateParam(LensConfConstants.QUERY_MAIL_NOTIFY, "false"); @@ -479,7 +459,7 @@ public class ITServerConfigTests extends BaseTestClass { } finally { if (sessionHandleString != null) { - sHelper.closeNewSession(sessionHandleString); + sHelper.closeSession(sessionHandleString); } } } @@ -488,32 +468,25 @@ public class ITServerConfigTests extends BaseTestClass { @Test(enabled = true) public void readInmemoryTwiceBeforePurgerTime() throws Exception { - sessionHandleString = lens.openSession(lens.getCurrentDB()); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); sHelper.setAndValidateParam(LensConfConstants.QUERY_MAIL_NOTIFY, "false"); - try { - - HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS, "500", - LensConfConstants.PURGE_INTERVAL, "10000"); - Util.changeConfig(map, confFilePath); - lens.restart(); - - QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); - LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); - Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Query did not succeed"); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS, "500", + LensConfConstants.PURGE_INTERVAL, "10000"); + Util.changeConfig(map, confFilePath); + lens.restart(); - Response response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); - Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); + QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.JDBC_CUBE_QUERY).getData(); + LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); + Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Query did not succeed"); - response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); - //Currently its throwing 500 which needs to be fixed. LENS-823 - Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + Response response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); - } finally { - lens.closeSession(); - } + response = qHelper.getResultSetResponse(queryHandle, "0", "100", sessionHandleString); + //Currently its throwing 500 which needs to be fixed. LENS-823 + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); } @@ -531,7 +504,7 @@ public class ITServerConfigTests extends BaseTestClass { lens.restart(); for (int i = 1; i <= maxSession; i++) { - sessionList.add(sHelper.openNewSession(user, pwd, lens.getCurrentDB())); + sessionList.add(sHelper.openSession(user, pwd, lens.getCurrentDB())); } Response response = sHelper.openSessionReturnResponse(user, pwd, lens.getCurrentDB(), null); @@ -540,7 +513,7 @@ public class ITServerConfigTests extends BaseTestClass { } finally { for (String session : sessionList) { if (session != null) { - sHelper.closeNewSession(session); + sHelper.closeSession(session); } } } @@ -564,8 +537,8 @@ public class ITServerConfigTests extends BaseTestClass { lens.restart(); for (int i = 0; i < maxSession; i++) { - sessionList1.add(sHelper.openNewSession(user1, pwd1, lens.getCurrentDB())); - sessionList2.add(sHelper.openNewSession(user2, pwd2, lens.getCurrentDB())); + sessionList1.add(sHelper.openSession(user1, pwd1, lens.getCurrentDB())); + sessionList2.add(sHelper.openSession(user2, pwd2, lens.getCurrentDB())); } for (int i = 0; i < maxSession; i++) { @@ -576,17 +549,17 @@ public class ITServerConfigTests extends BaseTestClass { Assert.assertEquals(response1.getStatus(), LensHttpStatus.TOO_MANY_REQUESTS.getStatusCode()); Assert.assertEquals(response2.getStatus(), LensHttpStatus.TOO_MANY_REQUESTS.getStatusCode()); - sHelper.closeNewSession(sessionList1.remove(1)); - sHelper.closeNewSession(sessionList2.remove(1)); + sHelper.closeSession(sessionList1.remove(1)); + sHelper.closeSession(sessionList2.remove(1)); - sessionList1.add(sHelper.openNewSession(user1, pwd1, lens.getCurrentDB())); - sessionList2.add(sHelper.openNewSession(user2, pwd2, lens.getCurrentDB())); + sessionList1.add(sHelper.openSession(user1, pwd1, lens.getCurrentDB())); + sessionList2.add(sHelper.openSession(user2, pwd2, lens.getCurrentDB())); } }finally { for (int i = 0; i < sessionList1.size(); i++) { - sHelper.closeNewSession(sessionList1.get(i)); - sHelper.closeNewSession(sessionList2.get(i)); + sHelper.closeSession(sessionList1.get(i)); + sHelper.closeSession(sessionList2.get(i)); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/66b7f47f/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java index b4846e3..8fea48b 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java @@ -78,7 +78,7 @@ public class ITCostTests extends BaseTestClass { public void initialize() throws Exception { servLens = ServiceManagerHelper.init(); logger.info("Creating a new Session"); - sessionHandleString = lens.openSession(lens.getCurrentDB()); + sessionHandleString = sHelper.openSession(lens.getCurrentDB()); sHelper.setAndValidateParam(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); } @@ -98,7 +98,7 @@ public class ITCostTests extends BaseTestClass { @AfterClass(alwaysRun = true) public void closeSession() throws Exception { logger.info("Closing Session"); - lens.closeSession(); + sHelper.closeSession(); } @BeforeGroups("user-cost-ceiling") @@ -221,41 +221,32 @@ public class ITCostTests extends BaseTestClass { HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "5", DriverConfig.PRIORITY_MAX_CONCURRENT, "HIGH=3"); - try { - Util.changeConfig(map, hiveDriverSitePath); - lens.restart(); - - List<QueryHandle> handleList = new ArrayList<QueryHandle>(); - for(int i=1; i<=5; i++){ - handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); - } - - QueryStatus s0 = qHelper.getQueryStatus(handleList.get(0)); - QueryStatus s1 = qHelper.getQueryStatus(handleList.get(1)); - QueryStatus s2 = qHelper.getQueryStatus(handleList.get(2)); - QueryStatus s3 = qHelper.getQueryStatus(handleList.get(3)); - QueryStatus s4 = qHelper.getQueryStatus(handleList.get(4)); + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + for(int i=1; i<=5; i++){ + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + } - Assert.assertEquals(s0.getStatus(), QueryStatus.Status.RUNNING); - Assert.assertEquals(s1.getStatus(), QueryStatus.Status.RUNNING); - Assert.assertEquals(s2.getStatus(), QueryStatus.Status.RUNNING); - Assert.assertEquals(s3.getStatus(), QueryStatus.Status.QUEUED); - Assert.assertEquals(s4.getStatus(), QueryStatus.Status.QUEUED); + QueryStatus s0 = qHelper.getQueryStatus(handleList.get(0)); + QueryStatus s1 = qHelper.getQueryStatus(handleList.get(1)); + QueryStatus s2 = qHelper.getQueryStatus(handleList.get(2)); + QueryStatus s3 = qHelper.getQueryStatus(handleList.get(3)); + QueryStatus s4 = qHelper.getQueryStatus(handleList.get(4)); - qHelper.waitForCompletion(handleList.get(0)); - Assert.assertEquals(qHelper.getQueryStatus(handleList.get(3)).getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s0.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s1.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s2.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s3.getStatus(), QueryStatus.Status.QUEUED); + Assert.assertEquals(s4.getStatus(), QueryStatus.Status.QUEUED); - qHelper.waitForCompletion(handleList.get(1)); - Assert.assertEquals(qHelper.getQueryStatus(handleList.get(4)).getStatus(), QueryStatus.Status.RUNNING); + qHelper.waitForCompletion(handleList.get(0)); + Assert.assertEquals(qHelper.getQueryStatus(handleList.get(3)).getStatus(), QueryStatus.Status.RUNNING); - for(QueryHandle q :handleList){ - LensQuery lq = qHelper.waitForCompletion(q); - Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - } + qHelper.waitForCompletion(handleList.get(1)); + Assert.assertEquals(qHelper.getQueryStatus(handleList.get(4)).getStatus(), QueryStatus.Status.RUNNING); - } finally { - Util.changeConfig(hiveDriverSitePath); - lens.restart(); + for(QueryHandle q :handleList){ + LensQuery lq = qHelper.waitForCompletion(q); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } } @@ -269,48 +260,39 @@ public class ITCostTests extends BaseTestClass { HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "5", DriverConfig.PRIORITY_MAX_CONCURRENT, "HIGH=3"); - try { - Util.changeConfig(map, hiveDriverSitePath); - lens.restart(); - - String session1 = sHelper.openNewSession("diff1", "diff1", lens.getCurrentDB()); - String session2 = sHelper.openNewSession("diff2", "diff2", lens.getCurrentDB()); - sHelper.setAndValidateParam(session1, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); - sHelper.setAndValidateParam(session2, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + String session1 = sHelper.openSession("diff1", "diff1", lens.getCurrentDB()); + String session2 = sHelper.openSession("diff2", "diff2", lens.getCurrentDB()); + sHelper.setAndValidateParam(session1, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + sHelper.setAndValidateParam(session2, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); - List<QueryHandle> handleList = new ArrayList<QueryHandle>(); - for (int i = 1; i <= 3; i++) { - handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); - handleList.add((QueryHandle) qHelper.executeQuery(query, "", session1).getData()); - handleList.add((QueryHandle) qHelper.executeQuery(query, "", session2).getData()); - } - Thread.sleep(50); - - List<QueryHandle> running = null, queued = null; - for (int t = 0; t < timeToWait; t = t + sleepTime) { + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + for (int i = 1; i <= 3; i++) { + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + handleList.add((QueryHandle) qHelper.executeQuery(query, "", session1).getData()); + handleList.add((QueryHandle) qHelper.executeQuery(query, "", session2).getData()); + } + Thread.sleep(50); - running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString, null, null, hiveDriver); - queued = qHelper.getQueryHandleList(null, "QUEUED", "all", sessionHandleString, null, null, hiveDriver); - logger.info("Running query count : " + running.size() + "\t Queued query count : " + queued.size()); + List<QueryHandle> running = null, queued = null; + for (int t = 0; t < timeToWait; t = t + sleepTime) { - if (running.isEmpty() && queued.isEmpty()) { - break; - } + running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString, null, null, hiveDriver); + queued = qHelper.getQueryHandleList(null, "QUEUED", "all", sessionHandleString, null, null, hiveDriver); + logger.info("Running query count : " + running.size() + "\t Queued query count : " + queued.size()); - Assert.assertTrue(running.size() < 4); - TimeUnit.SECONDS.sleep(sleepTime); + if (running.isEmpty() && queued.isEmpty()) { + break; } - Assert.assertTrue(running.isEmpty() && queued.isEmpty(), "Queries are taking very long time"); + Assert.assertTrue(running.size() < 4); + TimeUnit.SECONDS.sleep(sleepTime); + } - for (QueryHandle q : handleList) { - LensQuery lq = qHelper.waitForCompletion(q); - Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - } + Assert.assertTrue(running.isEmpty() && queued.isEmpty(), "Queries are taking very long time"); - } finally { - Util.changeConfig(hiveDriverSitePath); - lens.restart(); + for (QueryHandle q : handleList) { + LensQuery lq = qHelper.waitForCompletion(q); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } }
