Repository: lens Updated Branches: refs/heads/master 241603cf0 -> 08ce29785
LENS-1324 : Support more hooks in DriverQueryHook Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08ce2978 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08ce2978 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08ce2978 Branch: refs/heads/master Commit: 08ce29785f886b82b490f269a1ca4a6c6eff9eed Parents: 241603c Author: Puneet Gupta <puneetgu...@apache.org> Authored: Mon Oct 3 16:55:58 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon Oct 3 16:55:58 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/hive/HiveDriver.java | 1 + .../org/apache/lens/driver/jdbc/JDBCDriver.java | 1 + .../server/api/driver/AbstractLensDriver.java | 1 + .../lens/server/api/driver/DriverQueryHook.java | 54 +++++++++++++++++--- .../server/api/driver/NoOpDriverQueryHook.java | 44 ++++++++++++++-- .../query/cost/FactPartitionBasedQueryCost.java | 5 ++ .../lens/server/api/driver/MockDriver.java | 1 + .../server/api/user/MockDriverQueryHook.java | 34 +++++++++++- .../server/query/QueryExecutionServiceImpl.java | 12 ++++- .../lens/server/query/TestQueryService.java | 4 ++ 10 files changed, 144 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 84d9933..0d8810f 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -383,6 +383,7 @@ public class HiveDriver extends AbstractLensDriver { queryHook = driverConf.getClass( HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class ).newInstance(); + queryHook.setDriver(this); } catch (InstantiationException | IllegalAccessException e) { throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 3bf5e8f..f805ec6 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -407,6 +407,7 @@ public class JDBCDriver extends AbstractLensDriver { queryHook = this.conf.getClass( JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class ).newInstance(); + queryHook.setDriver(this); } catch (InstantiationException | IllegalAccessException e) { throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java index 03079e2..e498479 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java @@ -55,6 +55,7 @@ public abstract class AbstractLensDriver implements LensDriver { throw new LensException("Driver Type and Name can not be null or empty"); } fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString(); + noOpDriverQueryHook.setDriver(this); } /** http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java index da125bd..f8a9ee0 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java @@ -37,25 +37,65 @@ import org.apache.lens.server.api.query.QueryContext; * * This interface is expected to evolve for some time as more needs for hook are discovered * - + * Note: Note if the hook updates any configuration, same should be reflected in QueryContext + * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configuration is persisted and is available + * on server restarts and other bookkeeping needs. */ public interface DriverQueryHook { + /** - * Called just before launching the query on the selected driver. + * This setter method is called by the driver once hook instance is created. This driver information can be used while + * extracting driver specific information form the QueryContext. + * @param driver + */ + void setDriver(LensDriver driver); + + /** + * Called just before rewrite operation is tried on this driver + * * @param ctx * @throws LensException */ - void preLaunch(QueryContext ctx) throws LensException; + void preRewrite(AbstractQueryContext ctx) throws LensException; /** - * Called just after driver has been selected to execute a query. + * Called just after a successful rewrite operation is tried on this driver * - * Note: Note if this method updates any configuration, same should be reflected in QueryContext - * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configration is persisted and is available - * on server restarts and other bookkeeping needs. + * @param ctx + * @throws LensException + */ + void postRewrite(AbstractQueryContext ctx) throws LensException; + + /** + * Called just before estimate operation is tried on this driver + * Note : Estimate operation will be skipped if rewrite operation fails for this driver + * + * @param ctx + * @throws LensException + */ + void preEstimate(AbstractQueryContext ctx) throws LensException; + + /** + * Called just after a successful estimate operation is tried on this driver + * + * @param ctx + * @throws LensException + */ + void postEstimate(AbstractQueryContext ctx) throws LensException; + + /** + * Called just after driver has been selected to execute a query. * * @param ctx * @throws LensException */ void postDriverSelection(AbstractQueryContext ctx) throws LensException; + + /** + * Called just before launching the query on the selected driver. + * @param ctx + * @throws LensException + */ + void preLaunch(QueryContext ctx) throws LensException; + } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java index 8cd03cb..4f1f2eb 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java @@ -25,19 +25,55 @@ import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.QueryContext; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j public class NoOpDriverQueryHook implements DriverQueryHook { + + @Getter + private LensDriver driver; + @Override - public void preLaunch(QueryContext ctx) { - log.debug("Pre launch for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), - ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); + public void setDriver(LensDriver driver) { + this.driver = driver; + log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName()); + } + + @Override + public void preRewrite(AbstractQueryContext ctx) throws LensException { + log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(), + driver.getFullyQualifiedName()); + } + + @Override + public void postRewrite(AbstractQueryContext ctx) throws LensException { + log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), + ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); + } + + @Override + public void preEstimate(AbstractQueryContext ctx) throws LensException { + log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), + ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); + } + + @Override + public void postEstimate(AbstractQueryContext ctx) throws LensException { + log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}", + ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver), + ctx.getDriverQueryCost(driver)); } @Override public void postDriverSelection(AbstractQueryContext ctx) throws LensException { - log.debug("Post driver selection for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); + } + + @Override + public void preLaunch(QueryContext ctx) { + log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java index 792508b..4768550 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java @@ -64,4 +64,9 @@ public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBased public int compareTo(final FactPartitionBasedQueryCost o) { return new Double(partitionCost).compareTo(o.partitionCost); } + + @Override + public String toString() { + return getQueryCostType() + "(" + getEstimatedResourceUsage() + ")"; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java index 168b3cc..3c18ac7 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java @@ -99,6 +99,7 @@ public class MockDriver extends AbstractLensDriver { this.conf = conf; ioTestVal = conf.getInt("mock.driver.test.val", -1); this.conf.addResource(getDriverResourcePath("failing-query-driver-site.xml")); + getQueryHook().setDriver(this); } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java index 0c4a3f0..f70979a 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java @@ -34,6 +34,11 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook { public static final String UNSAVED_KEY_POST_SELECT = "TEST_UNSAVED__KEY_POST_SELECT"; public static final String UNSAVED_VALUE_POST_SELECT = "TEST_UNSAVED_VALUE_POST_SELECT"; + public static final String PRE_REWRITE = "PRE_REWRITE"; + public static final String POST_REWRITE = "POST_REWRITE"; + public static final String PRE_ESTIMATE = "PRE_ESTIMATE"; + public static final String POST_ESTIMATE = "POST_ESTIMATE"; + @Override public void preLaunch(QueryContext ctx) { super.preLaunch(ctx); @@ -46,9 +51,36 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook { //Updated both in driver config and LensConf(which gets persisted) ctx.getSelectedDriverConf().set(KEY_POST_SELECT, VALUE_POST_SELECT); - ctx.updateConf(new HashMap<String, String>(1){{put(KEY_POST_SELECT, VALUE_POST_SELECT); }}); + ctx.updateConf(new HashMap<String, String>(1) {{ + put(KEY_POST_SELECT, VALUE_POST_SELECT); + } + }); //Updated only in driver conf. ctx.getSelectedDriverConf().set(UNSAVED_KEY_POST_SELECT, UNSAVED_VALUE_POST_SELECT); } + + @Override + public void preRewrite(AbstractQueryContext ctx) throws LensException { + super.preRewrite(ctx); + ctx.getDriverConf(getDriver()).set(PRE_REWRITE, PRE_REWRITE); + } + + @Override + public void postRewrite(AbstractQueryContext ctx) throws LensException { + super.postRewrite(ctx); + ctx.getDriverConf(getDriver()).set(POST_REWRITE, POST_REWRITE); + } + + @Override + public void preEstimate(AbstractQueryContext ctx) throws LensException { + super.preEstimate(ctx); + ctx.getDriverConf(getDriver()).set(PRE_ESTIMATE, PRE_ESTIMATE); + } + + @Override + public void postEstimate(AbstractQueryContext ctx) throws LensException { + super.postEstimate(ctx); + ctx.getDriverConf(getDriver()).set(POST_ESTIMATE, POST_ESTIMATE); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 0bb106e..87d7cb0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -1689,12 +1689,17 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE acquire(ctx.getLensSessionIdentifier()); MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_GAUGE); + log.info("Calling preRewrite hook for driver {}", driver.getFullyQualifiedName()); + driver.getQueryHook().preRewrite(ctx); // 1. Rewrite for driver rewriterRunnable.run(); succeeded = rewriterRunnable.isSucceeded(); if (!succeeded) { failureCause = rewriterRunnable.getFailureCause(); cause = rewriterRunnable.getCause(); + } else { + log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName()); + driver.getQueryHook().postRewrite(ctx); } rewriteGauge.markSuccess(); @@ -1704,14 +1709,19 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE MethodMetricsContext estimateGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true, DRIVER_ESTIMATE_GAUGE); + log.info("Calling preEstimate hook for driver {}", driver.getFullyQualifiedName()); + driver.getQueryHook().preEstimate(ctx); estimateRunnable.run(); succeeded = estimateRunnable.isSucceeded(); - if (!succeeded) { failureCause = estimateRunnable.getFailureCause(); cause = estimateRunnable.getCause(); log.error("Estimate failed for driver {} cause: {}", driver, failureCause); + } else { + log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName()); + driver.getQueryHook().postEstimate(ctx); } + estimateGauge.markSuccess(); } else { log.error("Estimate skipped since rewrite failed for driver {} cause: {}", driver, failureCause); http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index f755508..3f71aef 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -766,6 +766,10 @@ public class TestQueryService extends LensJerseyTest { Thread.sleep(1000); } assertEquals(ctx.getSelectedDriverConf().get(KEY_PRE_LAUNCH), VALUE_PRE_LAUNCH); + assertEquals(ctx.getSelectedDriverConf().get(PRE_REWRITE), PRE_REWRITE); + assertEquals(ctx.getSelectedDriverConf().get(POST_REWRITE), POST_REWRITE); + assertEquals(ctx.getSelectedDriverConf().get(PRE_ESTIMATE), PRE_ESTIMATE); + assertEquals(ctx.getSelectedDriverConf().get(POST_ESTIMATE), POST_ESTIMATE); assertTrue(lensQuery.getSubmissionTime() > 0); assertTrue(lensQuery.getLaunchTime() > 0); assertTrue(lensQuery.getDriverStartTime() > 0);