Repository: incubator-lens Updated Branches: refs/heads/master 76218cb14 -> afc440005
LENS-329 : Use separate db connection pool for estimate queries in jdbc driver (Jaideep Dhok via amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/afc44000 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/afc44000 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/afc44000 Branch: refs/heads/master Commit: afc4400058c7ea832d79433a4cf56141d5941bba Parents: 76218cb Author: Jaideep Dhok <jd...@apache.org> Authored: Fri Mar 13 15:38:35 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Fri Mar 13 15:38:35 2015 +0530 ---------------------------------------------------------------------- .../jdbc/DataSourceConnectionProvider.java | 4 + .../org/apache/lens/driver/jdbc/JDBCDriver.java | 94 ++++++++++++++++- .../src/main/resources/jdbcdriver-default.xml | 63 +++++++++++ .../apache/lens/driver/jdbc/TestJdbcDriver.java | 104 +++++++++++++++++-- .../src/test/resources/jdbcdriver-site.xml | 57 ++++++++++ .../lens/server/query/TestQueryService.java | 7 +- .../src/test/resources/jdbcdriver-site.xml | 2 +- src/site/apt/admin/jdbcdriver-config.apt | 32 ++++-- 8 files changed, 340 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java index dcd6328..7b454fa 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java @@ -207,4 +207,8 @@ public class DataSourceConnectionProvider implements ConnectionProvider { LOG.info("Closed datasource connection provider"); } + protected final ComboPooledDataSource getDataSource(Configuration conf) { + return dataSourceMap.get(getDriverConfigfromConf(conf)); + } + } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index d67e1da..6f9dd5b 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -49,6 +49,7 @@ import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryRewriter; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -87,6 +88,11 @@ public class JDBCDriver implements LensDriver { /** The conf. */ private Configuration conf; + /** Configuration for estimate connection pool */ + private Configuration estimateConf; + /** Estimate connection provider */ + private ConnectionProvider estimateConnectionProvider; + /** * Data related to a query submitted to JDBCDriver. */ @@ -426,6 +432,7 @@ public class JDBCDriver implements LensDriver { DataSourceConnectionProvider.class, ConnectionProvider.class); try { connectionProvider = cpClass.newInstance(); + estimateConnectionProvider = cpClass.newInstance(); } catch (Exception e) { LOG.error("Error initializing connection provider: " + e.getMessage(), e); throw new LensException(e); @@ -548,6 +555,7 @@ public class JDBCDriver implements LensDriver { private static final String VALIDATE_GAUGE = "validate-thru-prepare"; private static final String COLUMNAR_SQL_REWRITE_GAUGE = "columnar-sql-rewrite"; private static final String JDBC_PREPARE_GAUGE = "jdbc-prepare-statement"; + @Override public QueryCost estimate(AbstractQueryContext qctx) throws LensException { MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true, @@ -622,7 +630,14 @@ public class JDBCDriver implements LensDriver { boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE, DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE); if (validateThroughPrepare) { - PreparedStatement stmt = prepareInternal(pContext); + PreparedStatement stmt = null; + try { + // Estimate queries need to get connection from estimate pool to make sure + // we are not blocked by data queries. + stmt = prepareInternal(pContext, getEstimateConnection(), true); + } catch (SQLException e) { + throw new LensException(e); + } if (stmt != null) { try { stmt.close(); @@ -633,6 +648,66 @@ public class JDBCDriver implements LensDriver { } } + // Get key used for estimate key config + protected String getEstimateKey(String jdbcKey) { + return JDBC_DRIVER_PFX + "estimate." + jdbcKey.substring(JDBC_DRIVER_PFX.length()); + } + + // If 'key' is set in conf, return its value. + // Otherwise, return value specified for fallBackKey + private String getKeyOrFallBack(Configuration conf, String key, String fallBackKey) { + String val = conf.get(key); + if (StringUtils.isBlank(val)) { + val = conf.get(fallBackKey); + } + return val; + } + + // Get connection config used by estimate pool. + protected final Configuration getEstimateConnectionConf() { + if (estimateConf == null) { + Configuration tmpConf = new Configuration(conf); + // Override JDBC settings in estimate conf, if set by user explicitly. Otherwise fall back to default JDBC pool + // config + tmpConf.set(JDBC_DB_URI, getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_DB_URI), JDBC_DB_URI)); + tmpConf.set(JDBC_DRIVER_CLASS, getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_DRIVER_CLASS), JDBC_DRIVER_CLASS)); + tmpConf.set(JDBC_USER, getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_USER), JDBC_USER)); + + String password = getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_PASSWORD), JDBC_PASSWORD); + /* We need to set password as empty string if it is not provided. Setting null on conf is not allowed */ + if (password == null) { + password = ""; + } + tmpConf.set(JDBC_PASSWORD, password); + + tmpConf.set(JDBC_POOL_MAX_SIZE, getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_POOL_MAX_SIZE), + JDBC_POOL_MAX_SIZE)); + tmpConf.set(JDBC_POOL_IDLE_TIME, getKeyOrFallBack(tmpConf, getEstimateKey(JDBC_POOL_IDLE_TIME), + JDBC_POOL_IDLE_TIME)); + tmpConf.set(JDBC_MAX_STATEMENTS_PER_CONNECTION, getKeyOrFallBack(tmpConf, + getEstimateKey(JDBC_MAX_STATEMENTS_PER_CONNECTION), JDBC_MAX_STATEMENTS_PER_CONNECTION)); + tmpConf.set(JDBC_GET_CONNECTION_TIMEOUT, getKeyOrFallBack(tmpConf, + getEstimateKey(JDBC_GET_CONNECTION_TIMEOUT), JDBC_GET_CONNECTION_TIMEOUT)); + + estimateConf = tmpConf; + } + return estimateConf; + } + + protected final Connection getEstimateConnection() throws SQLException { + return estimateConnectionProvider.getConnection(getEstimateConnectionConf()); + } + + // For tests + protected final ConnectionProvider getEstimateConnectionProvider() { + return estimateConnectionProvider; + } + + // For tests + protected final ConnectionProvider getConnectionProvider() { + return connectionProvider; + } + private final Map<QueryPrepareHandle, PreparedStatement> preparedQueries = new HashMap<QueryPrepareHandle, PreparedStatement>(); @@ -648,6 +723,21 @@ public class JDBCDriver implements LensDriver { throw new NullPointerException("Null driver query for " + pContext.getUserQuery()); } checkConfigured(); + return prepareInternal(pContext, getConnection(), false); + } + + + private PreparedStatement prepareInternal(AbstractQueryContext pContext, final Connection conn, + boolean checkConfigured) throws LensException { + // Caller might have already verified configured status and driver query, so we don't have + // to do this check twice. Caller must set checkConfigured to false in that case. + if (checkConfigured) { + if (pContext.getDriverQuery(this) == null) { + throw new NullPointerException("Null driver query for " + pContext.getUserQuery()); + } + checkConfigured(); + } + // Only create a prepared statement and then close it MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true, COLUMNAR_SQL_REWRITE_GAUGE); @@ -655,10 +745,8 @@ public class JDBCDriver implements LensDriver { sqlRewriteGauge.markSuccess(); MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true, JDBC_PREPARE_GAUGE); - Connection conn = null; PreparedStatement stmt = null; try { - conn = getConnection(); stmt = conn.prepareStatement(rewrittenQuery); if (stmt.getWarnings() != null) { throw new LensException(stmt.getWarnings()); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml index 03c5414..841dcac 100644 --- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml +++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml @@ -83,6 +83,69 @@ </description> </property> + <!-- Estimate query connection pool settings --> + <property> + <name>lens.driver.jdbc.estimate.driver.class</name> + <value></value> + <description>Type of JDBC driver used to connect backend database for estimate queries. If + This property is not specified, value for lens.driver.jdbc.driver.class will be used. + Override this property to tune estimate connection pool</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.db.uri</name> + <value></value> + <description>JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. + If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.db.user</name> + <value></value> + <description>The database user on whose behalf the connection is being made, for estimate queries. + If this property is unspecified, value for lens.driver.jdbc.db.user would be used. + Override this property to tune estimate connection pool</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.db.password</name> + <value></value> + <description>The database user's password, for estimate queries. If this property is unspecified, + value for lens.driver.jdbc.db.password would be used. + Override this property to tune estimate connection pool</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.max.size</name> + <value></value> + <description>Maximum number of concurrent connections allowed in pool, for estimate queries. + If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. + Override this property to tune estimate connection pool</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.idle.time</name> + <value></value> + <description>Maximum idle time in sec before a connection is closed, for estimate queries. + If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. + Override this property to tune estimate connection pool.</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.max.statements</name> + <value></value> + <description>Maximum number of prepared statements to cache per connection, for estimate queries. + If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.</description> + </property> + + <property> + <name>lens.driver.jdbc.estimate.get.connection.timeout</name> + <value></value> + <description>Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket + , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout + would be used. Override this property to tune estimate connection pool.</description> + </property> + <property> <name>lens.driver.jdbc.explain.keyword</name> <value>Explain</value> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 6c1d77e..f3be3fb 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -22,6 +22,7 @@ import static org.testng.Assert.*; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -44,6 +45,8 @@ import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.util.LensUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; @@ -56,11 +59,13 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.codahale.metrics.MetricRegistry; +import com.mchange.v2.c3p0.ComboPooledDataSource; /** * The Class TestJdbcDriver. */ public class TestJdbcDriver { + public static final Log LOG = LogFactory.getLog(TestJdbcDriver.class); /** The base conf. */ Configuration baseConf; @@ -130,10 +135,15 @@ public class TestJdbcDriver { * @throws Exception the exception */ synchronized void createTable(String table) throws Exception { - Connection conn = null; + createTable(table, null); + } + + synchronized void createTable(String table, Connection conn) throws Exception { Statement stmt = null; try { - conn = driver.getConnection(); + if (conn == null) { + conn = driver.getConnection(); + } stmt = conn.createStatement(); stmt.execute("CREATE TABLE " + table + " (ID INT)"); @@ -148,17 +158,23 @@ public class TestJdbcDriver { } } + + void insertData(String table) throws Exception { + insertData(table, null); + } + /** * Insert data. * * @param table the table * @throws Exception the exception */ - void insertData(String table) throws Exception { - Connection conn = null; + void insertData(String table, Connection conn) throws Exception { PreparedStatement stmt = null; try { - conn = driver.getConnection(); + if (conn == null) { + conn = driver.getConnection(); + } stmt = conn.prepareStatement("INSERT INTO " + table + " VALUES(?)"); for (int i = 0; i < 10; i++) { @@ -234,8 +250,8 @@ public class TestJdbcDriver { */ @Test public void testEstimate() throws Exception { - createTable("estimate_test"); // Create table - insertData("estimate_test"); // Insert some data into table + createTable("estimate_test", driver.getEstimateConnection()); // Create table + insertData("estimate_test", driver.getEstimateConnection()); // Insert some data into table String query1 = "SELECT * FROM estimate_test"; // Select query against existing table QueryCost cost = driver.estimate(createExplainContext(query1, baseConf)); Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0); @@ -265,8 +281,8 @@ public class TestJdbcDriver { */ @Test public void testEstimateGauges() throws Exception { - createTable("estimate_test_gauge"); // Create table - insertData("estimate_test_gauge"); // Insert some data into table + createTable("estimate_test_gauge", driver.getEstimateConnection()); // Create table + insertData("estimate_test_gauge", driver.getEstimateConnection()); // Insert some data into table String query1 = "SELECT * FROM estimate_test_gauge"; // Select query against existing table Configuration metricConf = new Configuration(baseConf); metricConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestJdbcDriver.class.getSimpleName()); @@ -346,8 +362,13 @@ public class TestJdbcDriver { */ @Test public void testPrepare() throws Exception { + // In this test we are testing both prepare and validate. Since in the test config + // We are using different DBs for estimate pool and query pool, we have to create + // tables in both DBs. createTable("prepare_test"); + createTable("prepare_test", driver.getEstimateConnection()); insertData("prepare_test"); + insertData("prepare_test", driver.getEstimateConnection()); final String query = "SELECT * from prepare_test"; PreparedQueryContext pContext = new PreparedQueryContext(query, "SA", baseConf, drivers); @@ -626,4 +647,69 @@ public class TestJdbcDriver { driver.closeQuery(handle); } + @Test + public void testEstimateConf() { + Configuration estimateConf = driver.getEstimateConnectionConf(); + assertNotNull(estimateConf); + assertTrue(estimateConf != driver.getConf()); + + // Validate overridden conf + assertEquals(estimateConf.get(JDBCDriverConfConstants.JDBC_USER), "estimateUser"); + assertEquals(estimateConf.get(JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE), "50"); + assertEquals(estimateConf.get(JDBCDriverConfConstants.JDBC_POOL_IDLE_TIME), "800"); + assertEquals(estimateConf.get(JDBCDriverConfConstants.JDBC_GET_CONNECTION_TIMEOUT), "25000"); + assertEquals(estimateConf.get(JDBCDriverConfConstants.JDBC_MAX_STATEMENTS_PER_CONNECTION), + "15"); + } + + @Test + public void testEstimateConnectionPool() throws Exception { + assertNotNull(driver.getEstimateConnectionProvider()); + assertTrue(driver.getEstimateConnectionProvider() != driver.getConnectionProvider()); + + ConnectionProvider connectionProvider = driver.getEstimateConnectionProvider(); + assertTrue(connectionProvider instanceof DataSourceConnectionProvider); + + DataSourceConnectionProvider estimateCp = (DataSourceConnectionProvider) connectionProvider; + DataSourceConnectionProvider queryCp = (DataSourceConnectionProvider) driver.getConnectionProvider(); + + assertTrue(estimateCp != queryCp); + + DataSourceConnectionProvider.DriverConfig estimateCfg = + estimateCp.getDriverConfigfromConf(driver.getEstimateConnectionConf()); + DataSourceConnectionProvider.DriverConfig queryCfg = + queryCp.getDriverConfigfromConf(driver.getConf()); + + LOG.info("@@@ ESTIMATE_CFG " + estimateCfg); + LOG.info("@@@ QUERY CFG " + queryCfg); + + // Get connection from each so that pools get initialized + try { + Connection estimateConn = estimateCp.getConnection(driver.getEstimateConnectionConf()); + estimateConn.close(); + } catch (SQLException e) { + // Ignore exception + LOG.error("Error getting connection from estimate pool", e); + } + + try { + Connection queryConn = queryCp.getConnection(driver.getConf()); + queryConn.close(); + } catch (SQLException e) { + LOG.error("Error getting connection from query pool", e); + } + + + ComboPooledDataSource estimatePool = estimateCp.getDataSource(driver.getEstimateConnectionConf()); + ComboPooledDataSource queryPool = queryCp.getDataSource(driver.getConf()); + + assertTrue(estimatePool != queryPool); + + // Validate config on estimatePool + assertEquals(estimatePool.getMaxPoolSize(), 50); + assertEquals(estimatePool.getMaxIdleTime(), 800); + assertEquals(estimatePool.getCheckoutTimeout(), 25000); + assertEquals(estimatePool.getMaxStatementsPerConnection(), 15); + } + } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml new file mode 100644 index 0000000..1dc0a85 --- /dev/null +++ b/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml @@ -0,0 +1,57 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + <property> + <name>lens.driver.jdbc.estimate.db.user</name> + <value>estimateUser</value> + </property> + + <!-- We have to use a different DB for estimate pool, because we are using an inmemory HSQL db. + There seem to be some issues regarding sharing of underlying inmemory db with different connection + pool. Whichever is constructed later is not able to get connections. --> + <property> + <name>lens.driver.jdbc.estimate.db.uri</name> + <value>jdbc:hsqldb:mem:jdbcTestDBEstimate</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.max.size</name> + <value>50</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.idle.time</name> + <value>800</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.get.connection.timeout</name> + <value>25000</value> + </property> + + <property> + <name>lens.driver.jdbc.estimate.pool.max.statements</name> + <value>15</value> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 7686762..0f20bee 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -1391,6 +1391,7 @@ public class TestQueryService extends LensJerseyTest { */ @Test public void testEstimateFailingNativeQuery() throws InterruptedException { + LOG.info("@#@#testEstimateFailingNativeQuery"); final WebTarget target = target().path("queryapi/queries"); // estimate native query @@ -1404,13 +1405,15 @@ public class TestQueryService extends LensJerseyTest { final EstimateResult result = target.request() .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), EstimateResult.class); + LOG.info("@#@#testEstimateFailingNativeQuery got response"); Assert.assertNotNull(result); Assert.assertNull(result.getCost()); Assert.assertTrue(result.isError()); Assert.assertTrue(result.getErrorMsg().contains("Driver :org.apache.lens.driver.hive.HiveDriver Cause :Error while" - + " compiling statement: FAILED: SemanticException [Error 10001]: Line 1:32 Table not found 'nonexist'")); + + " compiling statement: FAILED: SemanticException [Error 10001]: Line 1:32 Table not found 'nonexist'"), + result.getErrorMsg()); Assert.assertTrue(result.getErrorMsg().contains("Driver :org.apache.lens.driver.jdbc.JDBCDriver Cause :user" - + " lacks privilege or object not found: NONEXIST")); + + " lacks privilege or object not found: NONEXIST"), result.getErrorMsg()); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/lens-server/src/test/resources/jdbcdriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/jdbcdriver-site.xml b/lens-server/src/test/resources/jdbcdriver-site.xml index c5dbd86..752372a 100644 --- a/lens-server/src/test/resources/jdbcdriver-site.xml +++ b/lens-server/src/test/resources/jdbcdriver-site.xml @@ -28,7 +28,7 @@ </property> <property> <name>lens.driver.jdbc.db.uri</name> - <value>jdbc:hsqldb:/tmp/db-storage.db;MODE=MYSQL</value> + <value>jdbc:hsqldb:./target/db-storage.db;MODE=MYSQL</value> </property> <property> <name>lens.driver.jdbc.db.user</name> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/afc44000/src/site/apt/admin/jdbcdriver-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/jdbcdriver-config.apt b/src/site/apt/admin/jdbcdriver-config.apt index dc325f1..bdc3d49 100644 --- a/src/site/apt/admin/jdbcdriver-config.apt +++ b/src/site/apt/admin/jdbcdriver-config.apt @@ -46,20 +46,36 @@ Jdbc driver configuration *--+--+---+--+ |11|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.| *--+--+---+--+ -|12|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database| +|12|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|13|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set| +|13|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.| *--+--+---+--+ -|14|lens.driver.jdbc.get.connection.timeout|10000|Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket| +|14|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|15|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed| +|15|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|16|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool| +|16|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.| *--+--+---+--+ -|17|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection| +|17|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.| *--+--+---+--+ -|18|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries| +|18|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|19|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.| +|19|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.| +*--+--+---+--+ +|20|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database| +*--+--+---+--+ +|21|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set| +*--+--+---+--+ +|22|lens.driver.jdbc.get.connection.timeout|10000|Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket| +*--+--+---+--+ +|23|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed| +*--+--+---+--+ +|24|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool| +*--+--+---+--+ +|25|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection| +*--+--+---+--+ +|26|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries| +*--+--+---+--+ +|27|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.| *--+--+---+--+ The configuration parameters and their default values
