Repository: lens Updated Branches: refs/heads/master 06d968151 -> f21130992
LENS-1016 : Fix setting priority in context. Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f2113099 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f2113099 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f2113099 Branch: refs/heads/master Commit: f211309921d1ba32626df7d34e8211c24ab50e24 Parents: 06d9681 Author: Amareshwari Sriramadasu <amareshw...@gmail.com> Authored: Tue Apr 26 16:40:01 2016 +0530 Committer: Puneet <puneet.gu...@inmobi.com> Committed: Tue Apr 26 16:40:01 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/hive/HiveDriver.java | 17 +++++++++++++++-- .../apache/lens/driver/hive/TestHiveDriver.java | 4 ++-- .../server/api/query/AbstractQueryContext.java | 19 +++++++++---------- .../lens/server/api/query/QueryContext.java | 12 ------------ .../api/query/TestAbstractQueryContext.java | 11 ----------- .../server/query/QueryExecutionServiceImpl.java | 6 ++++-- .../lens/server/query/TestQueryService.java | 15 +++++++++++++++ 7 files changed, 45 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index aa37dcc..6eec3f7 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -801,12 +801,25 @@ public class HiveDriver extends AbstractLensDriver { @Override public Priority decidePriority(AbstractQueryContext ctx) { - if (whetherCalculatePriority && ctx.getDriverConf(this).get("mapred.job.priority") == null) { + return decidePriority(ctx, queryPriorityDecider); + } + + Priority decidePriority(AbstractQueryContext ctx, QueryPriorityDecider queryPriorityDecider) { + if (whetherCalculatePriority && ctx.getPriority() == null) { try { + // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call. + // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and + // priority is set based on correct cost. + if (ctx.getDriverQueryCost(this) == null) { + ctx.setDriverCost(this, this.estimate(ctx)); + } // Inside try since non-data fetching queries can also be executed by async method. - Priority priority = ctx.decidePriority(this, queryPriorityDecider); + Priority priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)); String priorityStr = priority.toString(); ctx.getDriverConf(this).set("mapred.job.priority", priorityStr); + Map<String, String> confUpdate = new HashMap<>(); + confUpdate.put("mapred.job.priority", priorityStr); + ctx.updateConf(confUpdate); log.info("set priority to {}", priority); return priority; } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index ba3f3d5..3f77b0c 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -887,8 +887,8 @@ public class TestHiveDriver { ctx.setOlapQuery(true); Priority priority = driver.decidePriority(ctx); assertEquals(priority, expected, "cost: " + ctx.getDriverQueryCost(driver) + "priority: " + priority); - assertEquals(ctx.decidePriority(driver, - alwaysNormalPriorityDecider), Priority.NORMAL); + assertEquals(ctx.getConf().get("mapred.job.priority"), priority.toString()); + assertEquals(driver.decidePriority(ctx, alwaysNormalPriorityDecider), Priority.NORMAL); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java index b568ffb..e160f58 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java @@ -36,7 +36,6 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.query.DriverSelectorQueryContext.DriverQueryContext; import org.apache.lens.server.api.query.cost.QueryCost; -import org.apache.lens.server.api.query.priority.QueryPriorityDecider; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; @@ -479,15 +478,15 @@ public abstract class AbstractQueryContext implements Serializable { hiveConf = null; } - public Priority decidePriority(LensDriver driver, QueryPriorityDecider queryPriorityDecider) throws LensException { - // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call. - // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and - // priority is set based on correct cost. - if (getDriverQueryCost(driver) == null) { - setDriverCost(driver, driver.estimate(this)); + /** + * Update conf. + * + * @param confoverlay the conf to set + */ + public void updateConf(Map<String, String> confoverlay) { + lensConf.getProperties().putAll(confoverlay); + for (Map.Entry<String, String> prop : confoverlay.entrySet()) { + this.conf.set(prop.getKey(), prop.getValue()); } - priority = queryPriorityDecider.decidePriority(getDriverQueryCost(driver)); - return priority; } - } http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 94b79d0..2a11f97 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -307,18 +307,6 @@ public class QueryContext extends AbstractQueryContext { return conf; } - /** - * Update conf. - * - * @param confoverlay the conf to set - */ - public void updateConf(Map<String, String> confoverlay) { - lensConf.getProperties().putAll(confoverlay); - for (Map.Entry<String, String> prop : confoverlay.entrySet()) { - this.conf.set(prop.getKey(), prop.getValue()); - } - } - public String getResultSetParentDir() { return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT); } http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java index 7e9fda9..36cc8fa 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java @@ -18,7 +18,6 @@ */ package org.apache.lens.server.api.query; -import static org.apache.lens.api.Priority.HIGH; import static org.apache.lens.server.api.LensConfConstants.*; import static org.apache.lens.server.api.LensServerAPITestUtil.getConfiguration; @@ -28,12 +27,10 @@ import java.io.*; import java.util.Arrays; import java.util.List; -import org.apache.lens.api.Priority; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.driver.MockDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metrics.LensMetricsRegistry; -import org.apache.lens.server.api.query.priority.MockQueryPriorityDecider; import org.apache.hadoop.conf.Configuration; @@ -99,14 +96,6 @@ public class TestAbstractQueryContext { } @Test - public void testPrioritySetting() throws LensException { - MockQueryContext ctx = new MockQueryContext(); - Priority p = ctx.decidePriority(ctx.getSelectedDriver(), new MockQueryPriorityDecider()); - assertEquals(p, HIGH); - assertEquals(ctx.getPriority(), HIGH); - } - - @Test public void testReadAndWriteExternal() throws Exception { Configuration conf = new Configuration(); List<LensDriver> drivers = MockQueryContext.getDrivers(conf); http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index d1d8e65..c72a890 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -39,6 +39,7 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.Priority; import org.apache.lens.api.error.ErrorCollection; import org.apache.lens.api.query.*; import org.apache.lens.api.query.QueryStatus.Status; @@ -1324,7 +1325,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE * @param ctx query context * @throws LensException the lens exception */ - private void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException { + void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException { logSegregationContext.setLogSegragationAndQueryId(ctx.getLogHandle()); MethodMetricsContext parallelCallGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false, PARALLEL_CALL_GAUGE); @@ -1419,7 +1420,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE ctx.setSelectedDriver(driver); QueryCost selectedDriverQueryCost = ctx.getDriverContext().getDriverQueryCost(driver); ctx.setSelectedDriverQueryCost(selectedDriverQueryCost); - driver.decidePriority(ctx); + Priority priority = driver.decidePriority(ctx); + ctx.setPriority(priority == null ? Priority.NORMAL : priority); selectGauge.markSuccess(); } finally { parallelCallGauge.markSuccess(); http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index df13ba2..6bf077d 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -253,6 +253,21 @@ public class TestQueryService extends LensJerseyTest { assertTrue(lensQuery.getFinishTime() > 0); } + @Test + public void testPriorityOnMockQuery() throws Exception { + String query = "select mock, fail from " + TEST_TABLE; + QueryContext ctx = queryService.createContext(query, null, new LensConf(), new Configuration(), 5000L); + ctx.setLensSessionIdentifier(lensSessionId.getPublicId().toString()); + queryService.acquire(lensSessionId); + try { + queryService.rewriteAndSelect(ctx); + } finally { + queryService.release(lensSessionId); + } + assertNotNull(ctx.getSelectedDriver()); + assertEquals(ctx.getPriority(), Priority.NORMAL); + } + // test with execute async post, get all queries, get query context, // get wrong uuid query