Repository: lens Updated Branches: refs/heads/master 3f03d29ba -> f4da1f0b1
LENS-996: Improvements in filters of query list api Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f4da1f0b Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f4da1f0b Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f4da1f0b Branch: refs/heads/master Commit: f4da1f0b1fead1fc91694acb3280f79b5050820d Parents: 3f03d29 Author: Rajat Khandelwal <pro...@apache.org> Authored: Fri Jul 29 14:27:43 2016 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Fri Jul 29 14:27:43 2016 +0530 ---------------------------------------------------------------------- .../lens/cli/commands/LensQueryCommands.java | 12 +-- .../apache/lens/cli/TestLensQueryCommands.java | 41 +++++----- .../java/org/apache/lens/client/LensClient.java | 7 +- .../org/apache/lens/client/LensStatement.java | 13 ++-- .../org/apache/lens/cube/metadata/DateUtil.java | 6 +- .../lens/cube/parse/BetweenTimeRangeWriter.java | 9 +-- .../apache/lens/cube/metadata/TestDateUtil.java | 5 ++ .../lens/cube/parse/TestTimeRangeWriter.java | 34 ++++----- .../server/api/query/QueryExecutionService.java | 9 ++- .../apache/lens/server/query/LensServerDAO.java | 69 +++++++++-------- .../server/query/QueryExecutionServiceImpl.java | 78 +++++++++++++++----- .../lens/server/query/QueryServiceResource.java | 36 ++++----- .../apache/lens/server/util/UtilityMethods.java | 2 +- .../apache/lens/server/query/TestLensDAO.java | 7 +- .../lens/server/query/TestQueryService.java | 17 +++-- 15 files changed, 194 insertions(+), 151 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java index af05951..4346c86 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java @@ -267,10 +267,8 @@ public class LensQueryCommands extends BaseLensCommand { @CliOption(key = {"name"}, mandatory = false, help = "<query-name>") String queryName, @CliOption(key = {"user"}, mandatory = false, help = "<user-who-submitted-query>") String user, @CliOption(key = {"driver"}, mandatory = false, help = "<driver-where-query-ran>") String driver, - @CliOption(key = {"fromDate"}, mandatory = false, unspecifiedDefaultValue = "-1", help - = "<submission-time-is-after>") long fromDate, - @CliOption(key = {"toDate"}, mandatory = false, unspecifiedDefaultValue = "" + Long.MAX_VALUE, help - = "<submission-time-is-before>") long toDate) { + @CliOption(key = {"fromDate"}, mandatory = false, help = "<submission-time-is-after>") String fromDate, + @CliOption(key = {"toDate"}, mandatory = false, help = "<submission-time-is-before>") String toDate) { List<QueryHandle> handles = getClient().getQueries(state, queryName, user, driver, fromDate, toDate); if (handles != null && !handles.isEmpty()) { return Joiner.on("\n").skipNulls().join(handles).concat("\n").concat("Total number of queries: " @@ -378,10 +376,8 @@ public class LensQueryCommands extends BaseLensCommand { public String getAllPreparedQueries( @CliOption(key = {"name"}, mandatory = false, help = "<query-name>") String queryName, @CliOption(key = {"user"}, mandatory = false, help = "<user-who-submitted-query>") String userName, - @CliOption(key = {"fromDate"}, mandatory = false, unspecifiedDefaultValue = "-1", help - = "<submission-time-is-after>") long fromDate, - @CliOption(key = {"toDate"}, mandatory = false, unspecifiedDefaultValue = "" + Long.MAX_VALUE, help - = "<submission-time-is-before>") long toDate) { + @CliOption(key = {"fromDate"}, mandatory = false, help = "<submission-time-is-after>") String fromDate, + @CliOption(key = {"toDate"}, mandatory = false, help = "<submission-time-is-before>") String toDate) { List<QueryPrepareHandle> handles = getClient().getPreparedQueries(userName, queryName, fromDate, toDate); if (handles != null && !handles.isEmpty()) { return Joiner.on("\n").skipNulls().join(handles); http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java index 0bf2a66..ae30e9f 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java @@ -141,8 +141,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest { @Test(dataProvider = "executeSyncQueryDP") public void executeSyncQuery(LensQueryCommands qCom, String sql, boolean shouldPass, boolean closeConn) throws Exception{ - assertEquals(qCom.getAllPreparedQueries("all", "", -1, -1), "No prepared queries"); - String successfulQueries = qCom.getAllQueries("SUCCESSFUL", null, "all", null, -1, Long.MAX_VALUE); + assertEquals(qCom.getAllPreparedQueries("all", "", null, null), "No prepared queries"); + String successfulQueries = qCom.getAllQueries("SUCCESSFUL", null, "all", null, null, null); int noOfSuccQueriesSoFar; if (successfulQueries.contains("No queries")) { noOfSuccQueriesSoFar = 0; @@ -172,7 +172,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { } // Wait for query to reach successful state int chkCount = 0; - while (!qCom.getAllQueries("SUCCESSFUL", null, "all", null, -1, Long.MAX_VALUE).contains( + while (!qCom.getAllQueries("SUCCESSFUL", null, "all", null, null, null).contains( "Total number of queries: " + (noOfSuccQueriesSoFar + 1))) { Thread.sleep(100); chkCount++; @@ -195,11 +195,11 @@ public class TestLensQueryCommands extends LensCliApplicationTest { public void preparedQuery(LensQueryCommands qCom) throws Exception { long submitTime = System.currentTimeMillis(); String sql = "cube select id, name from test_dim"; - String result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE); + String result = qCom.getAllPreparedQueries("testPreparedName", "all", String.valueOf(submitTime), null); assertEquals(result, "No prepared queries"); final String qh = qCom.prepare(sql, "testPreparedName"); - result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, System.currentTimeMillis()); + result = qCom.getAllPreparedQueries("testPreparedName", "all", String.valueOf(submitTime), "now"); assertEquals(qh, result); result = qCom.getPreparedStatus(qh); @@ -235,16 +235,16 @@ public class TestLensQueryCommands extends LensCliApplicationTest { log.debug("destroy result is " + result); assertEquals("Successfully destroyed " + qh, result); - result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE); + result = qCom.getAllPreparedQueries("testPreparedName", "all", String.valueOf(submitTime), null); assertEquals(result, "No prepared queries"); final String qh2 = qCom.explainAndPrepare(sql, "testPrepQuery3"); assertExplainOutput(qh2); - String handles = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, Long.MAX_VALUE); + String handles = qCom.getAllPreparedQueries("testPrepQuery3", "all", null, null); assertFalse(handles.contains("No prepared queries"), handles); - String handles2 = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, submitTime - 1); + String handles2 = qCom.getAllPreparedQueries("testPrepQuery3", "all", null, String.valueOf(submitTime - 1)); assertFalse(handles2.contains(qh), handles2); result = qCom.destroyPreparedQuery(handles); assertEquals("Successfully destroyed " + handles, result); @@ -320,7 +320,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String qh = qCom.executeQuery(sql, true, "testQuery1"); String user = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh))) .getQuery().getSubmittedUser(); - String result = qCom.getAllQueries("", "testQuery1", user, "", -1, Long.MAX_VALUE); + String result = qCom.getAllQueries("", "testQuery1", user, "", null, null); // this is because previous query has run two query handle will be there assertTrue(result.contains(qh), result); assertTrue(result.contains("Total number of queries")); @@ -339,7 +339,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { } // Check that query name searching is 'ilike' - String result2 = qCom.getAllQueries("", "query", "all", "", -1, Long.MAX_VALUE); + String result2 = qCom.getAllQueries("", "query", "all", "", null, null); assertTrue(result2.contains(qh), result2); assertTrue(qCom.getStatus(qh).contains("Status: SUCCESSFUL")); String details = qCom.getDetails(qh); @@ -350,10 +350,10 @@ public class TestLensQueryCommands extends LensCliApplicationTest { // Kill query is not tested as there is no deterministic way of killing a query - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", -1, Long.MAX_VALUE); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", null, null); assertTrue(result.contains(qh), result); - result = qCom.getAllQueries("FAILED", "", "all", "", -1, Long.MAX_VALUE); + result = qCom.getAllQueries("FAILED", "", "all", "", null, null); if (!result.contains("No queries")) { // Make sure valid query handles are returned String[] handles = StringUtils.split(result, "\n"); @@ -367,30 +367,31 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String queryName = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery() .getQueryName(); assertTrue("testQuery1".equalsIgnoreCase(queryName), queryName); - result = qCom.getAllQueries("", "", "", "", submitTime, System.currentTimeMillis()); + result = qCom.getAllQueries("", "", "", "", String.valueOf(submitTime), "now"); assertTrue(result.contains(qh), result); - result = qCom.getAllQueries("", "fooBar", "all", "", submitTime, System.currentTimeMillis()); + result = qCom.getAllQueries("", "fooBar", "all", "", String.valueOf(submitTime), "now"); assertTrue(result.contains("No queries"), result); - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", submitTime, System.currentTimeMillis()); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", String.valueOf(submitTime), "now"); assertTrue(result.contains(qh)); - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", submitTime - 5000, submitTime - 1); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", String.valueOf(submitTime - 5000), + String.valueOf(submitTime - 1)); // should not give query since its not in the range assertFalse(result.contains(qh)); // Filters on driver - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "hive/hive1", submitTime, - System.currentTimeMillis()); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "hive/hive1", String.valueOf(submitTime), "now"); assertTrue(result.contains(qh)); - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "DummyDriver", submitTime, System.currentTimeMillis()); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "DummyDriver", String.valueOf(submitTime), "now"); assertFalse(result.contains(qh)); try { // Should fail with bad request since fromDate > toDate - result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", submitTime + 5000, submitTime); + result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", String.valueOf(submitTime + 5000), + String.valueOf(submitTime)); fail("Call should have failed with BadRequestException, instead got " + result); } catch (BadRequestException exc) { // pass http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-client/src/main/java/org/apache/lens/client/LensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index d3ec97d..9a76118 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -307,8 +307,8 @@ public class LensClient implements AutoCloseable { return getLensStatement(query).getResultSet(); } - public List<QueryHandle> getQueries(String state, String queryName, String user, String driver, long fromDate, - long toDate) { + public List<QueryHandle> getQueries(String state, String queryName, String user, String driver, String fromDate, + String toDate) { return statement.getAllQueries(state, queryName, user, driver, fromDate, toDate); } @@ -663,7 +663,8 @@ public class LensClient implements AutoCloseable { return statement.destroyPrepared(queryPrepareHandle); } - public List<QueryPrepareHandle> getPreparedQueries(String userName, String queryName, long fromDate, long toDate) { + public List<QueryPrepareHandle> getPreparedQueries(String userName, String queryName, String fromDate, + String toDate) { return statement.getAllPreparedQueries(userName, queryName, fromDate, toDate); } http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java index 3cae717..64c24c7 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java @@ -459,15 +459,14 @@ public class LensStatement { * @param toDate the to date * @return the all queries */ - public List<QueryHandle> getAllQueries(String state, String queryName, String user, String driver, long fromDate, - long toDate) { + public List<QueryHandle> getAllQueries(String state, String queryName, String user, String driver, String fromDate, + String toDate) { WebTarget target = getQueryWebTarget(connection.buildClient()); - List<QueryHandle> handles = target.queryParam("sessionid", connection.getSessionHandle()) + return target.queryParam("sessionid", connection.getSessionHandle()) .queryParam("state", state).queryParam("queryName", queryName).queryParam("user", user) .queryParam("driver", driver).queryParam("fromDate", fromDate).queryParam("toDate", toDate).request() .get(new GenericType<List<QueryHandle>>() { }); - return handles; } /** @@ -479,13 +478,13 @@ public class LensStatement { * @param toDate the to date * @return the all prepared queries */ - public List<QueryPrepareHandle> getAllPreparedQueries(String userName, String queryName, long fromDate, long toDate) { + public List<QueryPrepareHandle> getAllPreparedQueries(String userName, String queryName, String fromDate, + String toDate) { Client client = connection.buildClient(); WebTarget target = getPreparedQueriesWebTarget(client); - List<QueryPrepareHandle> handles = target.queryParam("sessionid", connection.getSessionHandle()) + return target.queryParam("sessionid", connection.getSessionHandle()) .queryParam("user", userName).queryParam("queryName", queryName).queryParam("fromDate", fromDate) .queryParam("toDate", toDate).request().get(new GenericType<List<QueryPrepareHandle>>() {}); - return handles; } /** http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java index b82cd95..7717081 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -56,6 +56,7 @@ public final class DateUtil { public static final String UNIT; public static final Date MAX_DATE = new Date(Long.MAX_VALUE); public static final Date MIN_DATE = new Date(Long.MIN_VALUE); + static { StringBuilder sb = new StringBuilder(); String sep = ""; @@ -85,6 +86,7 @@ public final class DateUtil { + WSPACE + ")?" + "(" + QUANTITY + ")" + OPTIONAL_WSPACE + "(" + UNIT + "))?" + "(s?)"; public static final Pattern RELDATE_VALIDATOR = Pattern.compile(RELDATE_VALIDATOR_STR, Pattern.CASE_INSENSITIVE); + public static final Pattern TIMESTAMP_VALIDATOR = Pattern.compile("\\d{5,}"); public static final String YEAR_FMT = "[0-9]{4}"; public static final String MONTH_FMT = YEAR_FMT + "-[0-9]{2}"; @@ -131,7 +133,9 @@ public final class DateUtil { } public static Date resolveDate(String str, Date now) throws LensException { - if (RELDATE_VALIDATOR.matcher(str).matches()) { + if (TIMESTAMP_VALIDATOR.matcher(str).matches()) { + return new Date(Long.parseLong(str)); + } else if (RELDATE_VALIDATOR.matcher(str).matches()) { return resolveRelativeDate(str, now); } else { return resolveAbsoluteDate(str); http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java index bd19d4c..a5b26c4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java @@ -18,7 +18,6 @@ */ package org.apache.lens.cube.parse; -import java.util.Iterator; import java.util.Set; import java.util.TreeSet; @@ -46,14 +45,12 @@ public class BetweenTimeRangeWriter implements TimeRangeWriter { partStr.append(partFilter); partStr.append(")"); } else { - TreeSet<FactPartition> parts = new TreeSet<FactPartition>(); + TreeSet<FactPartition> parts = new TreeSet<>(); FactPartition first = null; - Iterator<FactPartition> it = rangeParts.iterator(); - while (it.hasNext()) { - FactPartition part = it.next(); + for (FactPartition part : rangeParts) { if (part.hasContainingPart()) { throw new LensException(LensCubeErrorCode.CANNOT_USE_TIMERANGE_WRITER.getLensErrorInfo(), - "Partition has containing part"); + "Partition has containing part"); } if (first == null) { first = part; http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestDateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestDateUtil.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestDateUtil.java index 7e239f3..8b3b4ba 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestDateUtil.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestDateUtil.java @@ -294,4 +294,9 @@ public class TestDateUtil { assertEquals(relativeToAbsolute("now.day -2 day", now), DateUtil.ABSDATE_PARSER.get().format(nowDayMinus2Days)); assertEquals(relativeToAbsolute("now.day -2 days", now), DateUtil.ABSDATE_PARSER.get().format(nowDayMinus2Days)); } + @Test + public void testTimestamp() throws LensException { + Date now = new Date(); + assertEquals(DateUtil.resolveDate(String.valueOf(now.getTime()+1), now), new Date(now.getTime()+1)); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java index 87e128f..748f92f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java @@ -34,6 +34,7 @@ import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.server.api.error.LensException; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import lombok.extern.slf4j.Slf4j; @@ -110,24 +111,21 @@ public abstract class TestTimeRangeWriter { } } - - @Test - public void testConsecutiveDayParts() throws LensException { - Set<FactPartition> answeringParts = new LinkedHashSet<FactPartition>(); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -1), DAILY, null, null)); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -2), DAILY, null, null)); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, 0), DAILY, null, null)); - + @DataProvider + public Object[][] formatDataProvider() { + return new Object[][] { + {null, }, + {DB_FORMAT, }, + }; + } + @Test(dataProvider = "formatDataProvider") + public void testConsecutiveDayParts(DateFormat format) throws LensException, InterruptedException { + Set<FactPartition> answeringParts = new LinkedHashSet<>(); + answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -1), DAILY, null, format)); + answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -2), DAILY, null, format)); + answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, 0), DAILY, null, format)); String whereClause = getTimerangeWriter().getTimeRangeWhereClause(null, "test", answeringParts); - validateConsecutive(whereClause, null); - - answeringParts = new LinkedHashSet<FactPartition>(); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -1), DAILY, null, DB_FORMAT)); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -2), DAILY, null, DB_FORMAT)); - answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, 0), DAILY, null, DB_FORMAT)); - - whereClause = getTimerangeWriter().getTimeRangeWhereClause(null, "test", answeringParts); - validateConsecutive(whereClause, DB_FORMAT); + validateConsecutive(whereClause, format); } @Test @@ -137,7 +135,7 @@ public abstract class TestTimeRangeWriter { String whereClause = getTimerangeWriter().getTimeRangeWhereClause(null, "test", answeringParts); validateSingle(whereClause, null); - answeringParts = new LinkedHashSet<FactPartition>(); + answeringParts = new LinkedHashSet<>(); answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, -1), DAILY, null, DB_FORMAT)); whereClause = getTimerangeWriter().getTimeRangeWhereClause(null, "test", answeringParts); validateSingle(whereClause, DB_FORMAT); http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java index 4f4d4aa..575a236 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java @@ -227,7 +227,8 @@ public interface QueryExecutionService extends LensService, SessionValidator { * Returns all the queries in the specified state, for the given user and matching query name. * * @param sessionHandle the session handle - * @param state return queries in this state. if null, all queries will be returned + * @param states return queries in these state. if null, all queries will be returned. Multiple states can + * be supplied separated by comma * @param user Get queries submitted by a specific user. * If this set to "all", queries of all users are returned * @param driver Get queries submitted on a specific driver. @@ -237,8 +238,8 @@ public interface QueryExecutionService extends LensService, SessionValidator { * @return List of query handles * @throws LensException the lens exception */ - List<QueryHandle> getAllQueries(LensSessionHandle sessionHandle, String state, String user, String driver, - String queryName, long fromDate, long toDate) throws LensException; + List<QueryHandle> getAllQueries(LensSessionHandle sessionHandle, String states, String user, String driver, + String queryName, String fromDate, String toDate) throws LensException; /** * Returns all the prepared queries for the specified user. If no user is passed, queries of all users will be @@ -255,7 +256,7 @@ public interface QueryExecutionService extends LensService, SessionValidator { * @throws LensException the lens exception */ List<QueryPrepareHandle> getAllPreparedQueries(LensSessionHandle sessionHandle, String user, String queryName, - long fromDate, long toDate) throws LensException; + String fromDate, String toDate) throws LensException; /** * Destroy a prepared query. http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index 1d6125c..47159b5 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -27,6 +27,7 @@ import java.util.List; import javax.sql.DataSource; import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.FinishedLensQuery; import org.apache.lens.server.util.UtilityMethods; @@ -156,7 +157,7 @@ public class LensServerDAO { /** * Find finished queries. * - * @param state the state + * @param states the state * @param user the user * @param driverName the driver's fully qualified Name * @param queryName the query name @@ -165,43 +166,45 @@ public class LensServerDAO { * @return the list * @throws LensException the lens exception */ - public List<QueryHandle> findFinishedQueries(String state, String user, String driverName, String queryName, - long fromDate, long toDate) throws LensException { - boolean addFilter = StringUtils.isNotBlank(state) || StringUtils.isNotBlank(user) - || StringUtils.isNotBlank(queryName); + public List<QueryHandle> findFinishedQueries(List<QueryStatus.Status> states, String user, String driverName, + String queryName, long fromDate, long toDate) throws LensException { StringBuilder builder = new StringBuilder("SELECT handle FROM finished_queries"); List<Object> params = null; - if (addFilter) { - builder.append(" WHERE "); - List<String> filters = new ArrayList<String>(3); - params = new ArrayList<Object>(3); - - if (StringUtils.isNotBlank(state)) { - filters.add("status=?"); - params.add(state); - } - - if (StringUtils.isNotBlank(user)) { - filters.add("submitter=?"); - params.add(user); + builder.append(" WHERE "); + List<String> filters = new ArrayList<String>(3); + params = new ArrayList<Object>(3); + + if (states != null && !states.isEmpty()) { + StringBuilder statusFilterBuilder = new StringBuilder("status in ("); + String sep = ""; + for(QueryStatus.Status status: states) { + statusFilterBuilder.append(sep).append("?"); + sep = ", "; + params.add(status.toString()); } + filters.add(statusFilterBuilder.append(")").toString()); + } - if (StringUtils.isNotBlank(queryName)) { - filters.add("queryname like ?"); - params.add("%" + queryName + "%"); - } + if (StringUtils.isNotBlank(user)) { + filters.add("submitter=?"); + params.add(user); + } - if (StringUtils.isNotBlank(driverName)) { - filters.add("lower(drivername)=?"); - params.add(driverName.toLowerCase()); - } + if (StringUtils.isNotBlank(queryName)) { + filters.add("queryname like ?"); + params.add("%" + queryName + "%"); + } - filters.add("submissiontime BETWEEN ? AND ?"); - params.add(fromDate); - params.add(toDate); - builder.append(StringUtils.join(filters, " AND ")); + if (StringUtils.isNotBlank(driverName)) { + filters.add("lower(drivername)=?"); + params.add(driverName.toLowerCase()); } + filters.add("submissiontime BETWEEN ? AND ?"); + params.add(fromDate); + params.add(toDate); + builder.append(StringUtils.join(filters, " AND ")); + ResultSetHandler<List<QueryHandle>> resultSetHandler = new ResultSetHandler<List<QueryHandle>>() { @Override public List<QueryHandle> handle(ResultSet resultSet) throws SQLException { @@ -221,11 +224,7 @@ public class LensServerDAO { QueryRunner runner = new QueryRunner(ds); String query = builder.toString(); try { - if (addFilter) { - return runner.query(query, resultSetHandler, params.toArray()); - } else { - return runner.query(query, resultSetHandler); - } + return runner.query(query, resultSetHandler, params.toArray()); } catch (SQLException e) { throw new LensException(e); } http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index fddd3e4..f5ea3db 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -44,6 +44,7 @@ import org.apache.lens.api.Priority; import org.apache.lens.api.error.ErrorCollection; import org.apache.lens.api.query.*; import org.apache.lens.api.query.QueryStatus.Status; +import org.apache.lens.cube.metadata.DateUtil; import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.server.BaseLensService; import org.apache.lens.server.LensServerConf; @@ -92,7 +93,8 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; - +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.*; import lombok.extern.slf4j.Slf4j; @@ -2551,21 +2553,38 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE * java.lang.String, java.lang.String, java.lang.String, java.lang.String, long, long) */ @Override - public List<QueryHandle> getAllQueries(LensSessionHandle sessionHandle, String state, String userName, String driver, - String queryName, long fromDate, long toDate) throws LensException { - validateTimeRange(fromDate, toDate); + public List<QueryHandle> getAllQueries(LensSessionHandle sessionHandle, String states, String userName, String driver, + String queryName, String fromDate, String toDate) throws LensException { + long fromTime = -1; + long toTime = Long.MAX_VALUE; + Date now = new Date(); + if (fromDate != null) { + fromTime = DateUtil.resolveDate(fromDate, now).getTime(); + } + if (toDate != null) { + toTime = DateUtil.resolveDate(toDate, now).getTime(); + } + validateTimeRange(fromTime, toTime); userName = UtilityMethods.removeDomain(userName); + Set<Status> statuses; try { acquire(sessionHandle); - Status status = null; try { - status = StringUtils.isBlank(state) ? null : Status.valueOf(state); + if (StringUtils.isNotBlank(states)) { + statuses = Sets.newHashSet(); + for(String state: states.split(",")) { + statuses.add(Status.valueOf(state.trim().toUpperCase())); + } + } else { + statuses = Sets.newHashSet(Status.values()); + } } catch (IllegalArgumentException e) { throw new BadRequestException("Bad state argument passed, possible values are " + Status.values(), e); } - boolean filterByStatus = status != null; - queryName = queryName.toLowerCase(); boolean filterByQueryName = StringUtils.isNotBlank(queryName); + if (filterByQueryName) { + queryName = queryName.toLowerCase(); + } if (StringUtils.isBlank(userName)) { userName = getSession(sessionHandle).getLoggedInUser(); @@ -2578,28 +2597,42 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE QueryHandle q = itr.next(); QueryContext context = allQueries.get(q); long querySubmitTime = context.getSubmissionTime(); - if ((filterByStatus && status != context.getStatus().getStatus()) + if ((!statuses.contains(context.getStatus().getStatus())) || (filterByQueryName && !context.getQueryName().toLowerCase().contains(queryName)) || (filterByDriver && !context.getSelectedDriver().getFullyQualifiedName().equalsIgnoreCase(driver)) || (!"all".equalsIgnoreCase(userName) && !userName.equalsIgnoreCase(context.getSubmittedUser())) - || (!(fromDate <= querySubmitTime && querySubmitTime <= toDate))) { + || (!(fromTime <= querySubmitTime && querySubmitTime < toTime))) { itr.remove(); } } // Unless user wants to get queries in 'non finished' state, get finished queries from DB as well - if (status == null || status == CANCELED || status == SUCCESSFUL || status == FAILED) { + List<Status> finishedStatusesQueried = null; + if (statuses.size() != Status.values().length) { + finishedStatusesQueried = Lists.newArrayList(); + for (Status status: statuses) { + switch(status) { + case CANCELED: + case SUCCESSFUL: + case FAILED: + finishedStatusesQueried.add(status); + break; + default: + break; + } + } + } + if (finishedStatusesQueried == null || !finishedStatusesQueried.isEmpty()) { if ("all".equalsIgnoreCase(userName)) { userName = null; } - List<QueryHandle> persistedQueries = lensServerDao.findFinishedQueries(state, userName, driver, queryName, - fromDate, toDate); + List<QueryHandle> persistedQueries = lensServerDao.findFinishedQueries(finishedStatusesQueried, userName, + driver, queryName, fromTime, toTime); if (persistedQueries != null && !persistedQueries.isEmpty()) { log.info("Adding persisted queries {}", persistedQueries.size()); all.addAll(persistedQueries); } } - return all; } finally { release(sessionHandle); @@ -2615,8 +2648,17 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE */ @Override public List<QueryPrepareHandle> getAllPreparedQueries(LensSessionHandle sessionHandle, String user, String queryName, - long fromDate, long toDate) throws LensException { - validateTimeRange(fromDate, toDate); + String fromDate, String toDate) throws LensException { + long fromTime = -1; + long toTime = Long.MAX_VALUE; + Date now = new Date(); + if (fromDate != null) { + fromTime = DateUtil.resolveDate(fromDate, now).getTime(); + } + if (toDate != null) { + toTime = DateUtil.resolveDate(toDate, now).getTime(); + } + validateTimeRange(fromTime, toTime); user = UtilityMethods.removeDomain(user); try { acquire(sessionHandle); @@ -2640,7 +2682,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } } long queryPrepTime = preparedQueryContext.getPreparedTime().getTime(); - if (fromDate <= queryPrepTime && queryPrepTime <= toDate) { + if (fromTime <= queryPrepTime && queryPrepTime < toTime) { continue; } itr.remove(); @@ -2808,7 +2850,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE // we will create an instance and read its state still. try { Class<? extends LensDriver> driverCls = (Class<? extends LensDriver>) Class.forName(driverClsName); - driver = (LensDriver) driverCls.newInstance(); + driver = driverCls.newInstance(); String[] driverTypeAndName = StringUtils.split(driverQualifiedName, '/'); driver.configure(conf, driverTypeAndName[0], driverTypeAndName[1]); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index 803bbe3..7d8d300 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -113,30 +113,30 @@ public class QueryServiceResource { * parameter to false. * * @param sessionid The sessionid in which queryName is working - * @param state If any state is passed, all the queries in that state will be returned, otherwise all queries will - * be returned. Possible states are {link QueryStatus.Status#values()} + * @param states If any state is passed, all the queries in that state will be returned, otherwise all queries will + * be returned. Possible states are {link QueryStatus.Status#values()}. Multiple states can be + * passed as comma separated string * @param queryName If any queryName is passed, all the queries containing the queryName will be returned, otherwise * all the queries will be returned * @param user Returns queries submitted by this user. If set to "all", returns queries of all users. By default, * returns queries of the current user. * @param driver Get queries submitted on a specific driver. * @param fromDate from date to search queries in a time range, the range is inclusive(submitTime >= fromDate) - * @param toDate to date to search queries in a time range, the range is inclusive(toDate >= submitTime) + * from date can be a long value indicating timestamp, or it can be in a format acceptable in + * time_range_in function. Notably: yyyy[-MM[-dd[-HH-[mm...]]]], or now based relative format + * @param toDate to date to search queries in a time range, the range is inclusive(toDate > submitTime) + * possible formats it can take is same as fromDate * @return List of {@link QueryHandle} objects */ @GET @Path("queries") @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN}) public List<QueryHandle> getAllQueries(@QueryParam("sessionid") LensSessionHandle sessionid, - @DefaultValue("") @QueryParam("state") String state, @DefaultValue("") @QueryParam("queryName") String queryName, - @DefaultValue("") @QueryParam("user") String user, @DefaultValue("") @QueryParam("driver") String driver, - @DefaultValue("-1") @QueryParam("fromDate") long fromDate, @DefaultValue("-1") @QueryParam("toDate") long toDate) + @QueryParam("state") String states, @QueryParam("queryName") String queryName, @QueryParam("user") String user, + @QueryParam("driver") String driver, @QueryParam("fromDate") String fromDate, @QueryParam("toDate") String toDate) throws LensException { validateSessionId(sessionid); - if (toDate == -1L) { - toDate = Long.MAX_VALUE; - } - return queryServer.getAllQueries(sessionid, state, user, driver, queryName, fromDate, toDate); + return queryServer.getAllQueries(sessionid, states, user, driver, queryName, fromDate, toDate); } /** @@ -228,15 +228,14 @@ public class QueryServiceResource { public APIResult cancelAllQueries(@QueryParam("sessionid") LensSessionHandle sessionid, @DefaultValue("") @QueryParam("state") String state, @DefaultValue("") @QueryParam("user") String user, @DefaultValue("") @QueryParam("queryName") String queryName, @DefaultValue("") @QueryParam("driver") String driver, - @DefaultValue("-1") @QueryParam("fromDate") long fromDate, @DefaultValue("-1") @QueryParam("toDate") long toDate) + @QueryParam("fromDate") String fromDate, @QueryParam("toDate") String toDate) throws LensException { validateSessionId(sessionid); int numCancelled = 0; List<QueryHandle> handles = null; boolean failed = false; try { - handles = getAllQueries(sessionid, state, queryName, user, driver, fromDate, - toDate == -1L ? Long.MAX_VALUE : toDate); + handles = getAllQueries(sessionid, state, queryName, user, driver, fromDate, toDate); for (QueryHandle handle : handles) { if (queryServer.cancelQuery(sessionid, handle)) { numCancelled++; @@ -276,12 +275,8 @@ public class QueryServiceResource { @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN}) public List<QueryPrepareHandle> getAllPreparedQueries(@QueryParam("sessionid") LensSessionHandle sessionid, @DefaultValue("") @QueryParam("user") String user, @DefaultValue("") @QueryParam("queryName") String queryName, - @DefaultValue("-1") @QueryParam("fromDate") long fromDate, @DefaultValue("-1") @QueryParam("toDate") long toDate) - throws LensException { + @QueryParam("fromDate") String fromDate, @QueryParam("toDate") String toDate) throws LensException { validateSessionId(sessionid); - if (toDate == -1L) { - toDate = Long.MAX_VALUE; - } return queryServer.getAllPreparedQueries(sessionid, user, queryName, fromDate, toDate); } @@ -347,14 +342,13 @@ public class QueryServiceResource { @Produces({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) public APIResult destroyPreparedQueries(@QueryParam("sessionid") LensSessionHandle sessionid, @DefaultValue("") @QueryParam("user") String user, @DefaultValue("") @QueryParam("queryName") String queryName, - @DefaultValue("-1") @QueryParam("fromDate") long fromDate, - @DefaultValue("-1") @QueryParam("toDate") long toDate) throws LensException { + @QueryParam("fromDate") String fromDate, @QueryParam("toDate") String toDate) throws LensException { validateSessionId(sessionid); int numDestroyed = 0; boolean failed = false; List<QueryPrepareHandle> handles = null; try { - handles = getAllPreparedQueries(sessionid, user, queryName, fromDate, toDate == -1L ? Long.MAX_VALUE : toDate); + handles = getAllPreparedQueries(sessionid, user, queryName, fromDate, toDate); for (QueryPrepareHandle prepared : handles) { if (queryServer.destroyPrepared(sessionid, prepared)) { numDestroyed++; http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java index 42d9fe0..2842ec3 100644 --- a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java +++ b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java @@ -77,7 +77,7 @@ public final class UtilityMethods { * @return the string */ public static String removeDomain(String username) { - if (username.contains("@")) { + if (username != null && username.contains("@")) { username = username.substring(0, username.indexOf("@")); } return username; http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java index 7679a06..74d9a1f 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java @@ -44,6 +44,7 @@ import org.codehaus.jackson.map.ObjectMapper; import org.testng.Assert; import org.testng.annotations.Test; +import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; /** @@ -140,9 +141,9 @@ public class TestLensDAO { } System.out.println("@@ State = " + queryContext.getStatus().getStatus().name()); - List<QueryHandle> daoTestQueryHandles = service.lensServerDao.findFinishedQueries(finishedLensQuery.getStatus(), - queryContext.getSubmittedUser(), queryContext.getSelectedDriver().getFullyQualifiedName(), "daotestquery1", -1L, - Long.MAX_VALUE); + List<QueryHandle> daoTestQueryHandles = service.lensServerDao.findFinishedQueries( + Lists.newArrayList(QueryStatus.Status.valueOf(finishedLensQuery.getStatus())), queryContext.getSubmittedUser(), + queryContext.getSelectedDriver().getFullyQualifiedName(), "daotestquery1", -1L, Long.MAX_VALUE); Assert.assertEquals(daoTestQueryHandles.size(), 1); Assert.assertEquals(daoTestQueryHandles.get(0).getHandleId().toString(), finishedHandle); service.closeSession(session); http://git-wip-us.apache.org/repos/asf/lens/blob/f4da1f0b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index e8aa223..9f40a01 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -338,19 +338,24 @@ public class TestQueryService extends LensJerseyTest { long finishedQueries = metricsSvc.getFinishedQueries(); int noOfQueriesBeforeExecution = queryService.allQueries.size(); + long before = System.currentTimeMillis(); QueryHandle theHandle = executeAndGetHandle(target(), Optional.of(lensSessionId), Optional.of("select ID from " + TEST_TABLE), Optional.<LensConf>absent(), mt); - // Get all queries - // XML - List<QueryHandle> allQueriesXML = target.queryParam("sessionid", lensSessionId).request(MediaType.APPLICATION_XML) - .get(new GenericType<List<QueryHandle>>() {}); - assertTrue(allQueriesXML.size() >= 1); - List<QueryHandle> allQueries = target.queryParam("sessionid", lensSessionId).request(mt) .get(new GenericType<List<QueryHandle>>() {}); assertTrue(allQueries.size() >= 1); assertTrue(allQueries.contains(theHandle)); + // time filter + allQueries = target.queryParam("sessionid", lensSessionId).queryParam("fromDate", before) + .queryParam("toDate", "now").request(mt).get(new GenericType<List<QueryHandle>>() {}); + assertEquals(allQueries.size(), 1); + assertTrue(allQueries.contains(theHandle)); + + // status filter + allQueries = target.queryParam("sessionid", lensSessionId).queryParam("state", "SUCCESSFUL, CANCELED") + .request(mt).get(new GenericType<List<QueryHandle>>() {}); + assertTrue(allQueries.size() >= 1); String queryXML = target.path(theHandle.toString()).queryParam("sessionid", lensSessionId) .request(MediaType.APPLICATION_XML).get(String.class);
