LENS-123 : Adds ability to load different instances of same driver class
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/114dab34 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/114dab34 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/114dab34 Branch: refs/heads/master Commit: 114dab34642929152230cd049f1436ad796bedc2 Parents: 404d451 Author: Puneet Gupta <puneet.k.gu...@gmail.com> Authored: Mon Nov 30 11:06:12 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon Nov 30 11:06:12 2015 +0530 ---------------------------------------------------------------------- .../lens/api/query/LensPreparedQuery.java | 8 +- .../org/apache/lens/api/query/LensQuery.java | 4 +- .../lens/cli/commands/LensQueryCommands.java | 2 +- .../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++ .../org/apache/lens/client/LensStatement.java | 2 +- .../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++ .../lens/driver/cube/TestMinCostSelector.java | 8 +- .../org/apache/lens/driver/es/ESDriver.java | 13 +- .../org/apache/lens/driver/es/ESDriverTest.java | 2 +- .../org/apache/lens/driver/hive/HiveDriver.java | 51 ++++--- .../apache/lens/driver/hive/TestHiveDriver.java | 4 +- .../lens/driver/hive/TestRemoteHiveDriver.java | 16 +- .../drivers/hive/hive1/hivedriver-site.xml | 49 +++++++ .../src/test/resources/hivedriver-site.xml | 49 ------- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 38 ++--- .../driver/jdbc/TestColumnarSQLRewriter.java | 6 +- .../apache/lens/driver/jdbc/TestJDBCFinal.java | 2 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 12 +- .../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 70 +++++++++ .../src/test/resources/jdbcdriver-site.xml | 70 --------- .../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++ lens-ml-lib/src/test/resources/lens-site.xml | 2 +- .../lib/query/TestAbstractFileFormatter.java | 2 +- .../lens/server/api/LensConfConstants.java | 20 ++- .../server/api/driver/AbstractLensDriver.java | 71 +++++++++ .../lens/server/api/driver/LensDriver.java | 13 +- .../server/api/query/AbstractQueryContext.java | 2 +- .../api/query/DriverSelectorQueryContext.java | 4 +- .../server/api/query/FinishedLensQuery.java | 15 +- .../server/api/query/PreparedQueryContext.java | 7 +- .../lens/server/api/query/QueryContext.java | 3 +- .../lens/server/api/driver/MockDriver.java | 14 +- .../lens/server/api/query/MockQueryContext.java | 2 +- .../api/query/TestAbstractQueryContext.java | 4 +- .../apache/lens/server/query/LensServerDAO.java | 10 +- .../server/query/QueryExecutionServiceImpl.java | 146 ++++++++++++++----- .../apache/lens/server/rewrite/RewriteUtil.java | 2 +- .../lens/server/session/LensSessionImpl.java | 15 +- .../src/main/resources/lensserver-default.xml | 2 +- .../org/apache/lens/server/LensJerseyTest.java | 3 + .../apache/lens/server/TestServerRestart.java | 10 +- .../lens/server/query/TestEventService.java | 2 + .../apache/lens/server/query/TestLensDAO.java | 2 +- .../lens/server/query/TestQueryConstraints.java | 60 +------- .../lens/server/query/TestQueryService.java | 38 +++-- .../lens/server/rewrite/TestRewriting.java | 17 ++- .../drivers/hive/hive1/hivedriver-site.xml | 85 +++++++++++ .../drivers/hive/hive2/hivedriver-site.xml | 85 +++++++++++ .../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 58 ++++++++ .../mock/fail1/failing-query-driver-site.xml | 32 ++++ .../mockHive/mockHive1/hivedriver-site.xml | 95 ++++++++++++ .../mockHive/mockHive2/hivedriver-site.xml | 95 ++++++++++++ .../resources/failing-query-driver-site.xml | 27 ---- .../src/test/resources/hivedriver-site.xml | 80 ---------- .../src/test/resources/jdbcdriver-site.xml | 55 ------- lens-server/src/test/resources/lens-site.xml | 4 +- src/site/apt/admin/config-server.apt | 37 ++++- src/site/apt/admin/config.apt | 2 +- src/site/apt/lenshome/install-and-run.apt | 18 ++- .../drivers/hive/hive1/hivedriver-site.xml | 57 ++++++++ .../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 50 +++++++ .../server/hivedriver-site.xml | 57 -------- .../server/jdbcdriver-site.xml | 50 ------- tools/conf-pseudo-distr/server/lens-site.xml | 2 +- .../drivers/hive/hive1/hivedriver-site.xml | 41 ++++++ .../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 50 +++++++ tools/conf/server/hivedriver-site.xml | 41 ------ tools/conf/server/jdbcdriver-site.xml | 50 ------- tools/conf/server/lens-site.xml | 2 +- 69 files changed, 1463 insertions(+), 722 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java index 9595ce9..f5bc35d 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java @@ -50,8 +50,8 @@ import lombok.NoArgsConstructor; * the prepared time * @param preparedUser * the prepared user - * @param selectedDriverClassName - * the selected driver class name + * @param selectedDriverName + * the selected driver's fully qualified name * @param driverQuery * the driver query * @param conf @@ -93,11 +93,11 @@ public class LensPreparedQuery { private String preparedUser; /** - * The selected driver class name. + * The selected driver's fully qualified name. */ @XmlElement @Getter - private String selectedDriverClassName; + private String selectedDriverName; /** * The driver query. http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java index 204ecee..23c4dec 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java @@ -46,7 +46,7 @@ import lombok.*; * the priority * @param isPersistent * the is persistent - * @param selectedDriverClassName + * @param selectedDriverName * the selected driver class name * @param driverQuery * the driver query @@ -121,7 +121,7 @@ public class LensQuery { */ @XmlElement @Getter - private String selectedDriverClassName; + private String selectedDriverName; /** * The driver query. http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java index 83b919b..fe9b84d 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java @@ -364,7 +364,7 @@ public class LensQueryCommands extends BaseLensCommand { .append("Prepare handle:").append(prepared.getPrepareHandle()).append("\n") .append("User:" + prepared.getPreparedUser()).append("\n") .append("Prepared at:").append(prepared.getPreparedTime()).append("\n") - .append("Selected driver :").append(prepared.getSelectedDriverClassName()).append("\n") + .append("Selected driver :").append(prepared.getSelectedDriverName()).append("\n") .append("Driver query:").append(prepared.getDriverQuery()).append("\n"); if (prepared.getConf() != null) { sb.append("Conf:").append(prepared.getConf().getProperties()).append("\n"); http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml new file mode 100644 index 0000000..f2aed88 --- /dev/null +++ b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>lens.driver.hive.connection.class</name> + <value>org.apache.lens.driver.hive.RemoteThriftConnection</value> + <description>The connection class from HiveDriver to HiveServer.</description> + </property> + + <property> + Â + <name>hive.metastore.local</name> + Â + <value>true</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>${project.build.directory}/hive/warehouse</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + + <property> + <name>hive.lock.manager</name> + <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value> + </property> + + <property> + <name>hive.server2.thrift.bind.host</name> + <value>localhost</value> + </property> + + <property> + <name>hive.server2.thrift.port</name> + <value>12345</value> + </property> + + <property> + <name>hive.server2.thrift.client.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>hive.server2.thrift.client.connect.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>lens.driver.test.key</name> + <value>set</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-client/src/main/java/org/apache/lens/client/LensStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java index 0a511f0..40e6d76 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java @@ -213,7 +213,7 @@ public class LensStatement { } } LensClient.getCliLooger().info("User query: '{}' was submitted to {}", query.getUserQuery(), - query.getSelectedDriverClassName()); + query.getSelectedDriverName()); if (query.getDriverQuery() != null) { LensClient.getCliLooger().info(" Driver query: '{}' and Driver handle: {}", query.getDriverQuery(), query.getDriverOpHandle()); http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml new file mode 100644 index 0000000..f2aed88 --- /dev/null +++ b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>lens.driver.hive.connection.class</name> + <value>org.apache.lens.driver.hive.RemoteThriftConnection</value> + <description>The connection class from HiveDriver to HiveServer.</description> + </property> + + <property> + Â + <name>hive.metastore.local</name> + Â + <value>true</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>${project.build.directory}/hive/warehouse</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + + <property> + <name>hive.lock.manager</name> + <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value> + </property> + + <property> + <name>hive.server2.thrift.bind.host</name> + <value>localhost</value> + </property> + + <property> + <name>hive.server2.thrift.port</name> + <value>12345</value> + </property> + + <property> + <name>hive.server2.thrift.client.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>hive.server2.thrift.client.connect.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>lens.driver.test.key</name> + <value>set</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java index 72f1497..8267229 100644 --- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java +++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java @@ -66,13 +66,13 @@ public class TestMinCostSelector { LensConf qconf = new LensConf(); MockDriver d1 = new MockDriver(); - d1.configure(conf); + d1.configure(conf, null, null); MockDriver d2 = new MockDriver(); - d2.configure(conf); + d2.configure(conf, null, null); MockFailDriver fd1 = new MockFailDriver(); - fd1.configure(conf); + fd1.configure(conf, null, null); MockFailDriver fd2 = new MockFailDriver(); - fd2.configure(conf); + fd2.configure(conf, null, null); drivers.add(d1); drivers.add(d2); http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java index 14d9f99..d166e43 100644 --- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java +++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java @@ -48,6 +48,7 @@ import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.commons.lang.Validate; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -62,13 +63,14 @@ import org.antlr.runtime.tree.Tree; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import lombok.extern.slf4j.Slf4j; /** * Driver for elastic search */ @Slf4j -public class ESDriver implements LensDriver { +public class ESDriver extends AbstractLensDriver { private static final AtomicInteger THID = new AtomicInteger(); private static final double STREAMING_PARTITION_COST = 0; @@ -317,10 +319,11 @@ public class ESDriver implements LensDriver { } @Override - public void configure(Configuration conf) throws LensException { + public void configure(Configuration conf, String driverType, String driverName) throws LensException { + super.configure(conf, driverType, driverName); this.conf = new Configuration(conf); this.conf.addResource("esdriver-default.xml"); - this.conf.addResource("esdriver-site.xml"); + this.conf.addResource(getDriverResourcePath("esdriver-site.xml")); config = new ESDriverConfig(this.conf); Class klass; try { @@ -341,10 +344,10 @@ public class ESDriver implements LensDriver { | InstantiationException | IllegalAccessException | InvocationTargetException e) { - log.error("ES driver cannot start!", e); + log.error("ES driver {} cannot start!", getFullyQualifiedName(), e); throw new LensException("Cannot start es driver", e); } - log.debug("ES Driver configured"); + log.info("ES Driver {} configured", getFullyQualifiedName()); asyncQueryPool = Executors.newCachedThreadPool(new ThreadFactory() { @Override public Thread newThread(Runnable runnable) { http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java index f453416..ab6f22b 100644 --- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java +++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java @@ -35,7 +35,7 @@ public abstract class ESDriverTest { public void beforeTest() throws LensException { initializeConfig(config); esDriverConfig = new ESDriverConfig(config); - driver.configure(config); + driver.configure(config, "es", "es1"); mockClientES = (MockClientES) driver.getESClient(); } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 19c4793..a84c679 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -50,6 +50,7 @@ import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf; import org.apache.lens.server.api.query.priority.QueryPriorityDecider; import org.apache.commons.lang.StringUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -65,6 +66,7 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; import com.google.common.collect.ImmutableSet; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -72,7 +74,7 @@ import lombok.extern.slf4j.Slf4j; * The Class HiveDriver. */ @Slf4j -public class HiveDriver implements LensDriver { +public class HiveDriver extends AbstractLensDriver { /** The Constant HIVE_CONNECTION_CLASS. */ public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class"; @@ -331,16 +333,18 @@ public class HiveDriver implements LensDriver { * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration) */ @Override - public void configure(Configuration conf) throws LensException { + public void configure(Configuration conf, String driverType, String driverName) throws LensException { + super.configure(conf, driverType, driverName); this.driverConf = new Configuration(conf); + String driverConfPath = getDriverResourcePath("hivedriver-site.xml"); this.driverConf.addResource("hivedriver-default.xml"); - this.driverConf.addResource("hivedriver-site.xml"); + this.driverConf.addResource(driverConfPath); // resources have to be added separately on hiveConf again because new HiveConf() overrides hive.* properties // from HiveConf this.hiveConf = new HiveConf(conf, HiveDriver.class); this.hiveConf.addResource("hivedriver-default.xml"); - this.hiveConf.addResource("hivedriver-site.xml"); + this.hiveConf.addResource(driverConfPath); connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); @@ -366,6 +370,7 @@ public class HiveDriver implements LensDriver { } queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf); selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf); + log.info("Hive driver {} configured successfully", getFullyQualifiedName()); } private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException { @@ -378,7 +383,7 @@ public class HiveDriver implements LensDriver { @Override public QueryCost estimate(AbstractQueryContext qctx) throws LensException { - log.info("Estimate: {}", qctx.getDriverQuery(this)); + log.info("{} Estimate: {}", getFullyQualifiedName(), qctx.getDriverQuery(this)); if (qctx.getDriverQuery(this) == null) { throw new NullPointerException("Null driver query for " + qctx.getUserQuery()); } @@ -413,7 +418,7 @@ public class HiveDriver implements LensDriver { // explain called again and again return (HiveQueryPlan) explainCtx.getDriverContext().getDriverQueryPlan(this); } - log.info("Explain: {}", explainCtx.getDriverQuery(this)); + log.info("{} Explain: {}", getFullyQualifiedName(), explainCtx.getDriverQuery(this)); Configuration explainConf = new Configuration(explainCtx.getDriverConf(this)); explainConf.setClassLoader(explainCtx.getConf().getClassLoader()); explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false); @@ -751,7 +756,7 @@ public class HiveDriver implements LensDriver { */ @Override public void close() { - log.info("CloseDriver"); + log.info("CloseDriver {}", getFullyQualifiedName()); // Close this driver and release all resources sessionLock.lock(); try { @@ -812,8 +817,8 @@ public class HiveDriver implements LensDriver { connection = new ExpirableConnection(tconn, connectionExpiryTimeout); thriftConnExpiryQueue.offer(connection); threadConnections.put(connectionKey, connection); - log.info("New thrift connection {} for thread: {} for user: {} connection ID={}", connectionClass, - Thread.currentThread().getId(), user, connection.getConnId()); + log.info("New thrift connection {} for thread: {} for user: {} connection ID={} on driver:{}", + connectionClass, Thread.currentThread().getId(), user, connection.getConnId(), getFullyQualifiedName()); } catch (Exception e) { throw new LensException(e); } @@ -885,7 +890,7 @@ public class HiveDriver implements LensDriver { context.unSetDriverPersistent(); hiveQuery = context.getSelectedDriverQuery(); } - log.info("Hive driver query:{}", hiveQuery); + log.info("Hive driver {} query:{}", getFullyQualifiedName(), hiveQuery); context.setSelectedDriverQuery(hiveQuery); } @@ -914,14 +919,15 @@ public class HiveDriver implements LensDriver { try { hiveSession = getClient().openSession(ctx.getClusterUser(), ""); lensToHiveSession.put(sessionDbKey, hiveSession); - log.info("New hive session for user: {} , lens session: {} hive session handle: {}", ctx.getClusterUser(), - sessionDbKey, hiveSession.getHandleIdentifier()); + log.info("New hive session for user: {} , lens session: {} , hive session handle: {} , driver : {}", + ctx.getClusterUser(), sessionDbKey, hiveSession.getHandleIdentifier(), getFullyQualifiedName()); for (LensEventListener<DriverEvent> eventListener : driverListeners) { try { eventListener.onEvent(new DriverSessionStarted(System.currentTimeMillis(), this, lensSession, hiveSession .getSessionId().toString())); } catch (Exception exc) { - log.error("Error sending driver start event to listener {}", eventListener, exc); + log.error("Error sending driver {} start event to listener {}", getFullyQualifiedName(), eventListener, + exc); } } } catch (Exception e) { @@ -1006,7 +1012,7 @@ public class HiveDriver implements LensDriver { return; } } catch (LensException e) { - log.debug("query handle: {} Not yet launched on driver", handle); + log.debug("query handle: {} Not yet launched on driver {}", handle, getFullyQualifiedName()); } Thread.sleep(pollInterval); timeSpent += pollInterval; @@ -1069,9 +1075,9 @@ public class HiveDriver implements LensDriver { QueryHandle qhandle = (QueryHandle) in.readObject(); OperationHandle opHandle = new OperationHandle((TOperationHandle) in.readObject()); hiveHandles.put(qhandle, opHandle); - log.debug("Hive driver recovered {}:{}", qhandle, opHandle); + log.debug("Hive driver {} recovered {}:{}", getFullyQualifiedName(), qhandle, opHandle); } - log.info("HiveDriver recovered {} queries", hiveHandles.size()); + log.info("Hive driver {} recovered {} queries", getFullyQualifiedName(), hiveHandles.size()); int numSessions = in.readInt(); for (int i = 0; i < numSessions; i++) { String lensId = in.readUTF(); @@ -1079,7 +1085,7 @@ public class HiveDriver implements LensDriver { TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6); lensToHiveSession.put(lensId, sHandle); } - log.info("HiveDriver recovered {} sessions", lensToHiveSession.size()); + log.info("Hive driver {} recovered {} sessions", getFullyQualifiedName(), lensToHiveSession.size()); } } @@ -1096,15 +1102,15 @@ public class HiveDriver implements LensDriver { for (Map.Entry<QueryHandle, OperationHandle> entry : hiveHandles.entrySet()) { out.writeObject(entry.getKey()); out.writeObject(entry.getValue().toTOperationHandle()); - log.debug("Hive driver persisted {}:{}", entry.getKey(), entry.getValue()); + log.debug("Hive driver {} persisted {}:{}", getFullyQualifiedName(), entry.getKey(), entry.getValue()); } - log.info("HiveDriver persisted {} queries ", hiveHandles.size()); + log.info("Hive driver {} persisted {} queries ", getFullyQualifiedName(), hiveHandles.size()); out.writeInt(lensToHiveSession.size()); for (Map.Entry<String, SessionHandle> entry : lensToHiveSession.entrySet()) { out.writeUTF(entry.getKey()); out.writeObject(entry.getValue().toTSessionHandle()); } - log.info("HiveDriver persisted {} sessions", lensToHiveSession.size()); + log.info("Hive driver {} persisted {} sessions", getFullyQualifiedName(), lensToHiveSession.size()); } } @@ -1166,14 +1172,15 @@ public class HiveDriver implements LensDriver { for (SessionHandle session : hiveSessionsToCheck) { if (isSessionInvalid(exc, session)) { // We have to expire previous session - log.info("Hive server session {} for lens session {} has become invalid", session, lensSession); + log.info("{} Hive server session {} for lens session {} has become invalid", getFullyQualifiedName(), session, + lensSession); sessionLock.lock(); try { // We should close all connections and clear the session map since // most likely all sessions are gone closeAllConnections(); lensToHiveSession.clear(); - log.info("Cleared all sessions"); + log.info("{} Cleared all sessions", getFullyQualifiedName()); } finally { sessionLock.unlock(); } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index 722a2da..11efd3c 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -112,13 +112,13 @@ public class TestHiveDriver { protected void createDriver() throws LensException { conf = new HiveConf(); - conf.addResource("hivedriver-site.xml"); + conf.addResource("drivers/hive/hive1/hivedriver-site.xml"); conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver = new HiveDriver(); - driver.configure(conf); + driver.configure(conf, "hive", "hive1"); drivers = Lists.<LensDriver>newArrayList(driver); System.out.println("TestHiveDriver created"); } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index 98edc28..ab5ada9 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -133,11 +133,11 @@ public class TestRemoteHiveDriver extends TestHiveDriver { protected void createDriver() throws LensException { dataBase = TestRemoteHiveDriver.class.getSimpleName().toLowerCase(); conf = new HiveConf(remoteConf); - conf.addResource("hivedriver-site.xml"); + conf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driver = new HiveDriver(); conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); - driver.configure(conf); + driver.configure(conf, "hive", "hive1"); drivers = Lists.<LensDriver>newArrayList(driver); System.out.println("TestRemoteHiveDriver created"); } @@ -155,7 +155,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { HiveConf thConf = new HiveConf(conf, TestRemoteHiveDriver.class); thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000); final HiveDriver thrDriver = new HiveDriver(); - thrDriver.configure(thConf); + thrDriver.configure(thConf, "hive", "hive1"); QueryContext ctx = createContext("USE " + dataBase, conf, thrDriver); thrDriver.execute(ctx); @@ -235,12 +235,12 @@ public class TestRemoteHiveDriver extends TestHiveDriver { public void testHiveDriverPersistence() throws Exception { System.out.println("@@@@ start_persistence_test"); HiveConf driverConf = new HiveConf(remoteConf, TestRemoteHiveDriver.class); - driverConf.addResource("hivedriver-site.xml"); + driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000); driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false); final HiveDriver oldDriver = new HiveDriver(); - oldDriver.configure(driverConf); + oldDriver.configure(driverConf, "hive", "hive1"); driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false); driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false); @@ -284,7 +284,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { ByteArrayInputStream driverInput = new ByteArrayInputStream(driverBytes.toByteArray()); HiveDriver newDriver = new HiveDriver(); newDriver.readExternal(new ObjectInputStream(driverInput)); - newDriver.configure(driverConf); + newDriver.configure(driverConf, "hive", "hive1"); driverInput.close(); ctx1 = readContext(ctx1bytes, newDriver); @@ -311,7 +311,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { boolean isDriverAvailable = (ctx.getSelectedDriver() != null); out.writeBoolean(isDriverAvailable); if (isDriverAvailable) { - out.writeUTF(ctx.getSelectedDriver().getClass().getName()); + out.writeUTF(ctx.getSelectedDriver().getFullyQualifiedName()); } } finally { out.flush(); @@ -340,7 +340,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { ctx.setConf(driver.getConf()); boolean driverAvailable = in.readBoolean(); if (driverAvailable) { - String clsName = in.readUTF(); + String driverQualifiedName = in.readUTF(); ctx.setSelectedDriver(driver); } } finally { http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml new file mode 100644 index 0000000..613938d --- /dev/null +++ b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -0,0 +1,49 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + Â + <name>hive.metastore.local</name> + Â + <value>true</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>${project.build.directory}/hive/warehouse</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + + <property> + <name>lens.query.result.parent.dir</name> + <value>target/lens-results</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/resources/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/resources/hivedriver-site.xml b/lens-driver-hive/src/test/resources/hivedriver-site.xml deleted file mode 100644 index 613938d..0000000 --- a/lens-driver-hive/src/test/resources/hivedriver-site.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<configuration> - - <property> - Â - <name>hive.metastore.local</name> - Â - <value>true</value> - </property> - - <property> - <name>hive.metastore.warehouse.dir</name> - <value>${project.build.directory}/hive/warehouse</value> - </property> - - <property> - <name>javax.jdo.option.ConnectionURL</name> - <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> - <description>JDBC connect string for a JDBC metastore</description> - </property> - - <property> - <name>lens.query.result.parent.dir</name> - <value>target/lens-results</value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index a8b980f..d3fa904 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -60,12 +60,14 @@ import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; + + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import com.google.common.collect.ImmutableSet; + import lombok.Getter; import lombok.NonNull; import lombok.Setter; @@ -75,7 +77,7 @@ import lombok.extern.slf4j.Slf4j; * This driver is responsible for running queries against databases which can be queried using the JDBC API. */ @Slf4j -public class JDBCDriver implements LensDriver { +public class JDBCDriver extends AbstractLensDriver { /** The Constant THID. */ public static final AtomicInteger THID = new AtomicInteger(); @@ -420,17 +422,17 @@ public class JDBCDriver implements LensDriver { return conf; } - /** - * Configure driver with {@link org.apache.hadoop.conf.Configuration} passed + /* + * (non-Javadoc) * - * @param conf The configuration object - * @throws LensException the lens exception + * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration) */ @Override - public void configure(Configuration conf) throws LensException { + public void configure(Configuration conf, String driverType, String driverName) throws LensException { + super.configure(conf, driverType, driverName); this.conf = new Configuration(conf); this.conf.addResource("jdbcdriver-default.xml"); - this.conf.addResource("jdbcdriver-site.xml"); + this.conf.addResource(getDriverResourcePath("jdbcdriver-site.xml")); init(conf); try { queryHook = this.conf.getClass( @@ -440,7 +442,7 @@ public class JDBCDriver implements LensDriver { throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); } configured = true; - log.info("JDBC Driver configured"); + log.info("JDBC Driver {} configured", getFullyQualifiedName()); } /** @@ -514,9 +516,9 @@ public class JDBCDriver implements LensDriver { DummyQueryRewriter.class, QueryRewriter.class); try { rewriter = queryRewriterClass.newInstance(); - log.info("Initialized :{}", queryRewriterClass); + log.info("{} Initialized :{}", getFullyQualifiedName(), queryRewriterClass); } catch (Exception e) { - log.error("Unable to create rewriter object", e); + log.error("{} Unable to create rewriter object", getFullyQualifiedName(), e); throw new LensException(e); } rewriter.init(conf); @@ -636,7 +638,7 @@ public class JDBCDriver implements LensDriver { explainQuery = rewrittenQuery.replaceAll("select ", "select " + explainKeyword + " "); } - log.info("Explain Query : {}", explainQuery); + log.info("{} Explain Query : {}", getFullyQualifiedName(), explainQuery); QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null, new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false); QueryResult result = null; @@ -873,7 +875,7 @@ public class JDBCDriver implements LensDriver { checkConfigured(); String rewrittenQuery = rewriteQuery(context); - log.info("Execute {}", context.getQueryHandle()); + log.info("{} Execute {}", getFullyQualifiedName(), context.getQueryHandle()); QueryResult result = executeInternal(context, rewrittenQuery); return result.getLensResultSet(true); @@ -919,7 +921,7 @@ public class JDBCDriver implements LensDriver { throw new LensException("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e); } queryContextMap.put(context.getQueryHandle(), jdbcCtx); - log.info("ExecuteAsync: {}", context.getQueryHandle()); + log.info("{} ExecuteAsync: {}", getFullyQualifiedName(), context.getQueryHandle()); } /** @@ -1032,7 +1034,7 @@ public class JDBCDriver implements LensDriver { context.setEndTime(System.currentTimeMillis()); } context.closeResult(); - log.info("Cancelled query: {}", handle); + log.info("{} Cancelled query : {}", getFullyQualifiedName(), handle); } return cancelResult; } @@ -1053,7 +1055,7 @@ public class JDBCDriver implements LensDriver { } finally { queryContextMap.remove(handle); } - log.info("Closed query {}", handle.getHandleId()); + log.info("{} Closed query {}", getFullyQualifiedName(), handle.getHandleId()); } /** @@ -1069,7 +1071,7 @@ public class JDBCDriver implements LensDriver { try { closeQuery(query); } catch (LensException e) { - log.warn("Error closing query : {}", query.getHandleId(), e); + log.warn("{} Error closing query : {}", getFullyQualifiedName(), query.getHandleId(), e); } } for (QueryPrepareHandle query : preparedQueries.keySet()) { @@ -1080,7 +1082,7 @@ public class JDBCDriver implements LensDriver { throw new LensException(); } } catch (LensException e) { - log.warn("Error closing prapared query : {}", query, e); + log.warn("{} Error closing prapared query : {}", getFullyQualifiedName(), query, e); } } } finally { http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java index cf795fa..c412cf0 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java @@ -135,7 +135,7 @@ public class TestColumnarSQLRewriter { @BeforeTest public void setup() throws Exception { conf.addResource("jdbcdriver-default.xml"); - conf.addResource("jdbcdriver-site.xml"); + conf.addResource("drivers/jdbc/jdbc1/jdbcdriver-site.xml"); qtest.init(conf); List<FieldSchema> factColumns = new ArrayList<>(); @@ -435,8 +435,8 @@ public class TestColumnarSQLRewriter { String actual = qtest.rewrite(query, conf, hconf); String expected = "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), " - + "week((time_dim___time_dim . day )), " - + "date(( time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), " + + "week(( time_dim___time_dim . day )), " + + "date((time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), " + "case when (sum(alias2) = 0 ) then 0.0 else sum(alias2) end as `dollars_sold` , " + "format(sum(alias3), 4 ), format(avg(alias4), '##################.###' ), " + "min(alias5), max(alias6) from (select sales_fact___fact.time_key, " http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java index 053e20d..4eee354 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java @@ -75,7 +75,7 @@ public class TestJDBCFinal { baseConf.set(JDBCDriverConfConstants.JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for "); driver = new JDBCDriver(); - driver.configure(baseConf); + driver.configure(baseConf, "jdbc", "jdbc1"); assertNotNull(driver); assertTrue(driver.configured); System.out.println("Driver configured!"); http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 425bd6f..2ad9fcb 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -88,7 +88,7 @@ public class TestJdbcDriver { hConf = new HiveConf(baseConf, this.getClass()); driver = new JDBCDriver(); - driver.configure(baseConf); + driver.configure(baseConf, "jdbc", "jdbc1"); assertNotNull(driver); assertTrue(driver.configured); @@ -309,12 +309,12 @@ public class TestJdbcDriver { metricConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestJdbcDriver.class.getSimpleName()); driver.estimate(createExplainContext(query1, metricConf)); MetricRegistry reg = LensMetricsRegistry.getStaticRegistry(); - + String driverQualifiledName = driver.getFullyQualifiedName(); Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-columnar-sql-rewrite", - "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-jdbc-prepare-statement", - "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-thru-prepare", - "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-jdbc-check-allowed-query"))); + "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-columnar-sql-rewrite", + "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-jdbc-prepare-statement", + "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-thru-prepare", + "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-jdbc-check-allowed-query"))); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml new file mode 100644 index 0000000..1202074 --- /dev/null +++ b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml @@ -0,0 +1,70 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + <property> + <name>lens.driver.jdbc.estimate.db.user</name> + <value>estimateUser</value> + </property> + + <!-- We have to use a different DB for estimate pool, because we are using an inmemory HSQL db. + There seem to be some issues regarding sharing of underlying inmemory db with different connection + pool. Whichever is constructed later is not able to get connections. --> + <property> + <name>lens.driver.jdbc.estimate.db.uri</name> + <value>jdbc:hsqldb:mem:jdbcTestDBEstimate</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.connection.properties</name> + <value>random_key=random_value</value> + <description>Connection properties for jdbc connection.</description> + </property> + + + <property> + <name>lens.driver.jdbc.estimate.pool.max.size</name> + <value>50</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.idle.time</name> + <value>800</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.get.connection.timeout</name> + <value>25000</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.max.statements</name> + <value>15</value> + </property> + + <property> + <name>lens.driver.jdbc.regex.replacement.values</name> + <value>weekofyear=week, to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval$2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)</value> + <description>Rewriting the HQL to optimized sql queries</description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml deleted file mode 100644 index 1202074..0000000 --- a/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml +++ /dev/null @@ -1,70 +0,0 @@ -<?xml version="1.0"?> -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<configuration> - <property> - <name>lens.driver.jdbc.estimate.db.user</name> - <value>estimateUser</value> - </property> - - <!-- We have to use a different DB for estimate pool, because we are using an inmemory HSQL db. - There seem to be some issues regarding sharing of underlying inmemory db with different connection - pool. Whichever is constructed later is not able to get connections. --> - <property> - <name>lens.driver.jdbc.estimate.db.uri</name> - <value>jdbc:hsqldb:mem:jdbcTestDBEstimate</value> - </property> - - <property> - <name>lens.driver.jdbc.estimate.connection.properties</name> - <value>random_key=random_value</value> - <description>Connection properties for jdbc connection.</description> - </property> - - - <property> - <name>lens.driver.jdbc.estimate.pool.max.size</name> - <value>50</value> - </property> - - <property> - <name>lens.driver.jdbc.estimate.pool.idle.time</name> - <value>800</value> - </property> - - <property> - <name>lens.driver.jdbc.estimate.get.connection.timeout</name> - <value>25000</value> - </property> - - <property> - <name>lens.driver.jdbc.estimate.pool.max.statements</name> - <value>15</value> - </property> - - <property> - <name>lens.driver.jdbc.regex.replacement.values</name> - <value>weekofyear=week, to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval$2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)</value> - <description>Rewriting the HQL to optimized sql queries</description> - </property> -</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml new file mode 100644 index 0000000..f2aed88 --- /dev/null +++ b/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>lens.driver.hive.connection.class</name> + <value>org.apache.lens.driver.hive.RemoteThriftConnection</value> + <description>The connection class from HiveDriver to HiveServer.</description> + </property> + + <property> + Â + <name>hive.metastore.local</name> + Â + <value>true</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>${project.build.directory}/hive/warehouse</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + + <property> + <name>hive.lock.manager</name> + <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value> + </property> + + <property> + <name>hive.server2.thrift.bind.host</name> + <value>localhost</value> + </property> + + <property> + <name>hive.server2.thrift.port</name> + <value>12345</value> + </property> + + <property> + <name>hive.server2.thrift.client.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>hive.server2.thrift.client.connect.retry.limit</name> + <value>3</value> + </property> + + <property> + <name>lens.driver.test.key</name> + <value>set</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-ml-lib/src/test/resources/lens-site.xml ---------------------------------------------------------------------- diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml index 3d5dbef..9be7850 100644 --- a/lens-ml-lib/src/test/resources/lens-site.xml +++ b/lens-ml-lib/src/test/resources/lens-site.xml @@ -25,7 +25,7 @@ <configuration> <property> <name>lens.server.drivers</name> - <value>org.apache.lens.driver.hive.HiveDriver</value> + <value>hive:org.apache.lens.driver.hive.HiveDriver</value> </property> <property> http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java ---------------------------------------------------------------------- diff --git a/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java b/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java index 40e1cdc..c877516 100644 --- a/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java +++ b/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java @@ -251,7 +251,7 @@ public abstract class TestAbstractFileFormatter { protected QueryContext createContext(Configuration conf, String queryName) { final LensDriver mockDriver = new MockDriver(); try { - mockDriver.configure(conf); + mockDriver.configure(conf, null, null); } catch (LensException e) { Assert.fail(e.getMessage()); } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 7ee0749..88e5a01 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -52,9 +52,9 @@ public final class LensConfConstants { public static final String METASTORE_PFX = "lens.metastore."; /** - * The Constant DRIVER_CLASSES. + * The Constant DRIVER_TYPES_AND_CLASSES */ - public static final String DRIVER_CLASSES = SERVER_PFX + "drivers"; + public static final String DRIVER_TYPES_AND_CLASSES = SERVER_PFX + "drivers"; /** * The Constant DRIVER_SELECTOR_CLASS. */ @@ -943,6 +943,21 @@ public final class LensConfConstants { public static final int DEFAULT_FETCH_COUNT_SAVED_QUERY_LIST = 20; /** + * This is the base directory where all drivers are available under lens-server's Conf directory. + */ + public static final String DRIVERS_BASE_DIR = "drivers"; + + /** + * Name of the property that holds the path of "conf" directory of server + */ + public static final String CONFIG_LOCATION = "config.location"; + + /** + * Default location of "conf" directory (wrt to lens-server/bin) + */ + public static final String DEFAULT_CONFIG_LOCATION = "../conf"; + + /** * The Constant RESULTSET_PURGE_ENABLED. */ public static final String RESULTSET_PURGE_ENABLED = SERVER_PFX + "resultset.purge.enabled"; @@ -981,5 +996,4 @@ public final class LensConfConstants { * The Constant DEFAULT_HDFS_OUTPUT_RETENTION. */ public static final String DEFAULT_HDFS_OUTPUT_RETENTION = "1 day"; - } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java new file mode 100644 index 0000000..55f1535 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.driver; + + +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; + +import org.apache.hadoop.conf.Configuration; + +import lombok.Getter; + +/** + * Abstract class for Lens Driver Implementations. Provides default + * implementations and some utility methods for drivers + */ +public abstract class AbstractLensDriver implements LensDriver { + /** + * Separator used for constructing fully qualified name and driver resource path + */ + private static final char SEPARATOR = '/'; + + /** + * Driver's fully qualified name ( Example hive/hive1, jdbc/mysql1) + */ + @Getter + private String fullyQualifiedName = null; + + @Override + public void configure(Configuration conf, String driverType, String driverName) throws LensException { + if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) { + throw new LensException("Driver Type and Name can not be null or empty"); + } + fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString(); + } + + /** + * Gets the path (relative to lens server's conf location) for the driver resource in the system. This is a utility + * method that can be used by extending driver implementations to build path for their resources. + * + * @param resourceName + * @return + */ + protected String getDriverResourcePath(String resourceName) { + return new StringBuilder(LensConfConstants.DRIVERS_BASE_DIR).append(SEPARATOR).append(getFullyQualifiedName()) + .append(SEPARATOR).append(resourceName).toString(); + } + + @Override + public String toString() { + return getFullyQualifiedName(); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index a5a60d7..c220884 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -37,7 +37,6 @@ import com.google.common.collect.ImmutableSet; * The Interface LensDriver. */ public interface LensDriver extends Externalizable { - /** * Get driver configuration */ @@ -47,9 +46,11 @@ public interface LensDriver extends Externalizable { * Configure driver with {@link Configuration} passed. * * @param conf The configuration object + * @param driverType Type of the driver (Example: hive, jdbc, el) + * @param driverName Name of this driver * @throws LensException the lens exception */ - void configure(Configuration conf) throws LensException; + void configure(Configuration conf, String driverType, String driverName) throws LensException; /** * Estimate the cost of execution for given query. @@ -199,4 +200,12 @@ public interface LensDriver extends Externalizable { * null is never returned. */ ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies(); + + + /** + * @return fully qualified name of this driver. This should be unique for each driver instance. This name can be used + * for referring to the driver while logging, persisting and restoring driver details,etc. + * (Examples: hive/hive1, jdbc/mysql1 ) + */ + String getFullyQualifiedName(); } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java index 0c980a2..2f20113 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java @@ -256,7 +256,7 @@ public abstract class AbstractQueryContext implements Serializable { String expMsg = LensUtil.getCauseMessage(e); driverQueryContext.setDriverQueryCostEstimateError(e); failureCause = new StringBuilder("Driver :") - .append(driver.getClass().getName()) + .append(driver.getFullyQualifiedName()) .append(" Cause :") .append(expMsg) .toString(); http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java index feac938..5ff59bd 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java @@ -70,7 +70,7 @@ public class DriverSelectorQueryContext { String metricId = ctx.driverSpecificConf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY); if (!StringUtils.isBlank(metricId)) { ctx.driverSpecificConf.set(LensConfConstants.QUERY_METRIC_DRIVER_STACK_NAME, - metricId + "-" + driver.getClass().getSimpleName()); + metricId + "-" + driver.getFullyQualifiedName()); } ctx.setQuery(userQuery); driverQueryContextMap.put(driver, ctx); @@ -198,7 +198,7 @@ public class DriverSelectorQueryContext { log.error("Setting driver plan failed for driver {}", driver, e); String expMsg = LensUtil.getCauseMessage(e); driverQueryContext.setDriverQueryPlanGenerationError(e); - detailedFailureCause.append("\n Driver :").append(driver.getClass().getName()); + detailedFailureCause.append("\n Driver :").append(driver.getFullyQualifiedName()); detailedFailureCause.append(" Cause :" + expMsg); if (failureCause != null && !failureCause.equals(expMsg)) { useBuilder = true; http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java index 89053aa..a57a6e4 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java @@ -159,11 +159,11 @@ public class FinishedLensQuery { private String queryName; /** - * The selected driver class name. + * The selected driver's fully qualified name. */ @Getter @Setter - private String driverClass; + private String driverName; @Getter private LensDriver selectedDriver; @@ -197,14 +197,14 @@ public class FinishedLensQuery { } this.selectedDriver = ctx.getSelectedDriver(); if (null != ctx.getSelectedDriver()) { - this.driverClass = ctx.getSelectedDriver().getClass().getName(); + this.driverName = ctx.getSelectedDriver().getFullyQualifiedName(); } } public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) { - if (null == selectedDriver && null != driverClass) { - selectedDriver = getDriverFromClassName(drivers); + if (null == selectedDriver && null != driverName) { + selectedDriver = getDriverFromName(drivers); } QueryContext qctx = @@ -223,12 +223,11 @@ public class FinishedLensQuery { return qctx; } - private LensDriver getDriverFromClassName(Collection<LensDriver> drivers) { + private LensDriver getDriverFromName(Collection<LensDriver> drivers) { Iterator<LensDriver> iterator = drivers.iterator(); while (iterator.hasNext()) { LensDriver driver = iterator.next(); - if (driverClass.equals(driver.getClass().getName())) { - //TODO : LENS-123 - Ability to load different instances of same driver class + if (driverName.equals(driver.getFullyQualifiedName())) { return driver; } } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index b6f669b..0b08459 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -147,10 +147,9 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye * @return the lens prepared query */ public LensPreparedQuery toPreparedQuery() { - return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser, - getDriverContext().getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getClass() - .getCanonicalName() : null, getDriverContext().getSelectedDriverQuery(), - lensConf); + return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser, getDriverContext() + .getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getFullyQualifiedName() : null, + getDriverContext().getSelectedDriverQuery(), lensConf); } public String getQueryHandleString() { http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 9b491d1..b637665 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -306,8 +306,7 @@ public class QueryContext extends AbstractQueryContext { */ public LensQuery toLensQuery() { return new LensQuery(queryHandle, userQuery, super.getSubmittedUser(), priority, isPersistent, - getSelectedDriver() != null ? getSelectedDriver().getClass() - .getCanonicalName() : null, + getSelectedDriver() != null ? getSelectedDriver().getFullyQualifiedName() : null, getSelectedDriverQuery(), status, resultSetPath, driverOpHandle, lensConf, submissionTime, launchTime, driverStatus.getDriverStartTime(), http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java index 2d86589..a20cf47 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java @@ -40,6 +40,7 @@ import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.hadoop.conf.Configuration; + import org.apache.hive.service.cli.ColumnDescriptor; import com.beust.jcommander.internal.Sets; @@ -48,7 +49,7 @@ import com.google.common.collect.ImmutableSet; /** * The Class MockDriver. */ -public class MockDriver implements LensDriver { +public class MockDriver extends AbstractLensDriver { private static AtomicInteger mockDriverId = new AtomicInteger(); /** @@ -77,7 +78,7 @@ public class MockDriver implements LensDriver { @Override public String toString() { - return "MockDriver:" + driverId; + return getFullyQualifiedName()+":"+driverId; } @Override @@ -91,10 +92,15 @@ public class MockDriver implements LensDriver { * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration) */ @Override - public void configure(Configuration conf) throws LensException { + public void configure(Configuration conf, String driverType, String driverName) throws LensException { this.conf = conf; ioTestVal = conf.getInt("mock.driver.test.val", -1); - this.conf.addResource("failing-query-driver-site.xml"); + this.conf.addResource(getDriverResourcePath("failing-query-driver-site.xml")); + } + + @Override + public String getFullyQualifiedName() { + return "mock/fail1"; } /** http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java index fd6b560..7768917 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java @@ -54,7 +54,7 @@ public class MockQueryContext extends QueryContext { public static List<LensDriver> getDrivers(Configuration conf) throws LensException { List<LensDriver> drivers = Lists.newArrayList(); MockDriver d = new MockDriver(); - d.configure(conf); + d.configure(conf, null, null); drivers.add(d); return drivers; } http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java index 02b652e..a37a4c8 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java @@ -53,7 +53,7 @@ public class TestAbstractQueryContext { String uniqueMetridId = ctx.getConf().get(QUERY_METRIC_UNIQUE_ID_CONF_KEY); assertNotNull(uniqueMetridId); assertEquals(ctx.getSelectedDriverConf().get(QUERY_METRIC_DRIVER_STACK_NAME), - uniqueMetridId + "-" + MockDriver.class.getSimpleName()); + uniqueMetridId + "-" + new MockDriver().getFullyQualifiedName()); } @Test @@ -70,7 +70,7 @@ public class TestAbstractQueryContext { ctx.estimateCostForDrivers(); MetricRegistry reg = LensMetricsRegistry.getStaticRegistry(); assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.TestAbstractQueryContext-MockDriver-driverEstimate"))); + "lens.MethodMetricGauge.TestAbstractQueryContext-"+new MockDriver().getFullyQualifiedName()+"-driverEstimate"))); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index b9dd286..d8e654d 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -85,7 +85,7 @@ public class LensServerDAO { + "userquery varchar(10000) not null," + "submitter varchar(255) not null," + "starttime bigint, " + "endtime bigint," + "result varchar(255)," + "status varchar(255), " + "metadata varchar(100000), " + "rows int, " + "filesize bigint, " + "errormessage varchar(10000), " + "driverstarttime bigint, " - + "driverendtime bigint, " + "driverclass varchar(10000), " + + "driverendtime bigint, " + "drivername varchar(10000), " + "queryname varchar(255), " + "submissiontime bigint" + ")"; try { QueryRunner runner = new QueryRunner(ds); @@ -109,7 +109,7 @@ public class LensServerDAO { Connection conn = null; String sql = "insert into finished_queries (handle, userquery,submitter," + "starttime,endtime,result,status,metadata,rows,filesize," - + "errormessage,driverstarttime,driverendtime, driverclass, queryname, submissiontime)" + + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime)" + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { conn = getConnection(); @@ -117,7 +117,7 @@ public class LensServerDAO { runner.update(conn, sql, query.getHandle(), query.getUserQuery(), query.getSubmitter(), query.getStartTime(), query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(), query.getRows(), query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(), - query.getDriverClass(), query.getQueryName(), query.getSubmissionTime()); + query.getDriverName(), query.getQueryName(), query.getSubmissionTime()); conn.commit(); } finally { DbUtils.closeQuietly(conn); @@ -158,7 +158,7 @@ public class LensServerDAO { * * @param state the state * @param user the user - * @param driverName the driverClass + * @param driverName the driver's fully qualified Name * @param queryName the query name * @param fromDate the from date * @param toDate the to date @@ -192,7 +192,7 @@ public class LensServerDAO { } if (StringUtils.isNotBlank(driverName)) { - filters.add("lower(driverclass)=?"); + filters.add("lower(drivername)=?"); params.add(driverName.toLowerCase()); }