Repository: lens Updated Branches: refs/heads/master 9b7541bcb -> dce5a812d
LENS-383 :Add connection and read timeouts on rest api calls in lens client Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dce5a812 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dce5a812 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dce5a812 Branch: refs/heads/master Commit: dce5a812ddef01800bd83e229d6ccbad4ddee18a Parents: 9b7541b Author: Puneet Gupta <puneet.k.gu...@gmail.com> Authored: Tue May 3 10:06:28 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Tue May 3 10:06:28 2016 +0530 ---------------------------------------------------------------------- .../apache/lens/client/LensClientConfig.java | 8 ++ .../org/apache/lens/client/LensConnection.java | 13 ++- .../org/apache/lens/client/LensStatement.java | 41 +++++++-- .../src/main/resources/lens-client-default.xml | 12 +++ .../org/apache/lens/client/TestLensClient.java | 95 +++++++++++++++++--- .../server/MockQueryExecutionServiceImpl.java | 74 +++++++++++++++ src/site/apt/admin/config.apt | 4 +- src/site/apt/user/client-config.apt | 16 ++-- 8 files changed, 239 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java index 6a35a5e..b703e13 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java @@ -82,6 +82,14 @@ public class LensClientConfig extends Configuration { public static final String WS_FILTER_IMPL_SFX = ".ws.filter.impl"; + public static final String READ_TIMEOUT_MILLIS = CLIENT_PFX + "read.timeout.millis"; + + public static final int DEFAULT_READ_TIMEOUT_MILLIS = 300000; //5 mins + + public static final String CONNECTION_TIMEOUT_MILLIS = CLIENT_PFX + "connection.timeout.millis"; + + public static final int DEFAULT_CONNECTION_TIMEOUT_MILLIS = 60000; //60 secs + /** * Get the username from config * http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensConnection.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index 0c2557c..4951866 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -18,6 +18,8 @@ */ package org.apache.lens.client; +import static org.apache.lens.client.LensClientConfig.*; + import java.net.ConnectException; import java.util.HashMap; import java.util.Iterator; @@ -39,6 +41,7 @@ import org.apache.lens.api.StringList; import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; import org.apache.lens.client.exceptions.LensClientServerConnectionException; +import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataMultiPart; @@ -120,7 +123,15 @@ public class LensConnection { while (itr.hasNext()) { cb.register(itr.next()); } - return cb.build(); + Client client = cb.build(); + + //Set Timeouts + LensClientConfig config = params.getConf(); + client.property(ClientProperties.CONNECT_TIMEOUT, config.getInt(CONNECTION_TIMEOUT_MILLIS, + DEFAULT_CONNECTION_TIMEOUT_MILLIS)); + client.property(ClientProperties.READ_TIMEOUT, config.getInt(READ_TIMEOUT_MILLIS, DEFAULT_READ_TIMEOUT_MILLIS)); + + return client; } private WebTarget getSessionWebTarget() { http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java index f06bcd1..3cae717 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java @@ -18,6 +18,7 @@ */ package org.apache.lens.client; +import java.net.SocketTimeoutException; import java.util.List; import javax.ws.rs.client.Client; @@ -228,16 +229,20 @@ public class LensStatement { return mp; } + public void waitForQueryToComplete(QueryHandle handle) { + waitForQueryToComplete(handle, true); + } + /** * Wait for query to complete. * * @param handle the handle */ - public void waitForQueryToComplete(QueryHandle handle) { + void waitForQueryToComplete(QueryHandle handle, boolean retryOnTimeout) { LensClient.getCliLogger().info("Query handle: {}", handle); - LensQuery queryDetails = getQuery(handle); + LensQuery queryDetails = retryOnTimeout ? getQueryWithRetryOnTimeout(handle) : getQuery(handle); while (queryDetails.getStatus().queued()) { - queryDetails = getQuery(handle); + queryDetails = retryOnTimeout ? getQueryWithRetryOnTimeout(handle) : getQuery(handle); LensClient.getCliLogger().debug("Query {} status: {}", handle, queryDetails.getStatus()); try { Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval()); @@ -253,7 +258,7 @@ public class LensStatement { } while (!queryDetails.getStatus().finished() && !(queryDetails.getStatus().getStatus().equals(Status.CLOSED))) { - queryDetails = getQuery(handle); + queryDetails = retryOnTimeout ? getQueryWithRetryOnTimeout(handle) : getQuery(handle); LensClient.getCliLogger().info("Query Status:{} ", queryDetails.getStatus()); try { Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval()); @@ -299,7 +304,33 @@ public class LensStatement { .get(LensQuery.class); } catch (Exception e) { log.error("Failed to get query status, cause:", e); - throw new IllegalStateException("Failed to get query status, cause:" + e.getMessage()); + throw new IllegalStateException("Failed to get query status, cause:" + e.getMessage(), e); + } + } + + LensQuery getQueryWithRetryOnTimeout(QueryHandle handle) { + while (true) { + try { + return getQuery(handle); + } catch (Exception e) { + if (isExceptionDueToSocketTimeout(e)) { + log.warn("Could not get query status. Encountered socket timeout. Retrying..."); + continue; + } else { + throw e; + } + } + } + } + + static boolean isExceptionDueToSocketTimeout(Throwable err) { + if (err == null) { + return false; + } + if (err instanceof SocketTimeoutException) { + return true; + } else { + return isExceptionDueToSocketTimeout(err.getCause()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/resources/lens-client-default.xml ---------------------------------------------------------------------- diff --git a/lens-client/src/main/resources/lens-client-default.xml b/lens-client/src/main/resources/lens-client-default.xml index 5a578a7..96506ac 100644 --- a/lens-client/src/main/resources/lens-client-default.xml +++ b/lens-client/src/main/resources/lens-client-default.xml @@ -52,6 +52,18 @@ <description>Interval at which query progress will be polled. Interval has to be given in milliseconds</description> </property> <property> + <name>lens.client.connection.timeout.millis</name> + <value>60000</value> + <description>This is the maximum amount of time a client is blocked for making the initial connection. The Default + value for this property is 60 seconds.</description> + </property> + <property> + <name>lens.client.read.timeout.millis</name> + <value>300000</value> + <description>This is the maximum amount of time a client read operation is blocked waiting for data. The default + value of this property is 5 mins.</description> + </property> + <property> <name>lens.cli.json.pretty</name> <value>false</value> <description>Should CLI try to prettify the JSON of an object before priting.</description> http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java index 6e638f1..ff3b888 100644 --- a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java +++ b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java @@ -18,6 +18,9 @@ */ package org.apache.lens.client; +import static org.apache.lens.client.LensStatement.isExceptionDueToSocketTimeout; +import static org.apache.lens.server.MockQueryExecutionServiceImpl.ENABLE_SLEEP_FOR_GET_QUERY_OP; + import static org.testng.Assert.*; import java.io.File; @@ -29,11 +32,15 @@ import javax.xml.datatype.DatatypeFactory; import org.apache.lens.api.APIResult; import org.apache.lens.api.metastore.*; +import org.apache.lens.api.query.LensQuery; import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryHandleWithResultSet; +import org.apache.lens.client.exceptions.LensAPIException; import org.apache.lens.client.exceptions.LensClientIOException; import org.apache.lens.client.resultset.ResultSet; import org.apache.lens.server.LensAllApplicationJerseyTest; -import org.apache.lens.server.api.LensConfConstants; + +import org.apache.hadoop.hive.conf.HiveConf; import org.testng.Assert; import org.testng.annotations.*; @@ -43,25 +50,29 @@ import lombok.extern.slf4j.Slf4j; @Test(groups = "unit-test") @Slf4j public class TestLensClient extends LensAllApplicationJerseyTest { - private static final String TEST_DB = TestLensClient.class.getSimpleName(); - @Override - protected int getTestPort() { - return 10056; - } + private LensClient client; + + private static final String TEST_DB = TestLensClient.class.getSimpleName(); @Override protected URI getBaseUri() { return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build(); } - private LensClient client; + @Override + public HiveConf getServerConf() { + HiveConf conf = super.getServerConf(); + //Use MockQueryExecutionServiceImpl as QueryExecutionService for client tests + conf.set("lens.server.query.service.impl", "org.apache.lens.server.MockQueryExecutionServiceImpl"); + return conf; + } @BeforeTest public void setUp() throws Exception { super.setUp(); - client = new LensClient(); + client = new LensClient(createLensClientConfigWithServerUrl()); client.createDatabase(TEST_DB, true); assertTrue(client.setDatabase(TEST_DB)); @@ -121,6 +132,8 @@ public class TestLensClient extends LensAllApplicationJerseyTest { result = client.closeConnection(); assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + + super.tearDown(); } /** @@ -128,11 +141,10 @@ public class TestLensClient extends LensAllApplicationJerseyTest { */ @Test public void testClient() throws Exception { - LensClientConfig lensClientConfig = new LensClientConfig(); + LensClientConfig lensClientConfig = createLensClientConfigWithServerUrl(); lensClientConfig.setLensDatabase(TEST_DB); Assert.assertEquals(lensClientConfig.getLensDatabase(), TEST_DB); - lensClientConfig.set(LensConfConstants.SERVER_BASE_URL, "http://localhost:" + getTestPort() + "/lensapi"); LensClient client = new LensClient(lensClientConfig); Assert.assertEquals(client.getCurrentDatabae(), TEST_DB, "current database"); @@ -232,4 +244,67 @@ public class TestLensClient extends LensAllApplicationJerseyTest { private void compare(String[] actualArr, String[] expectedArr) { assertTrue(Arrays.equals(actualArr, expectedArr)); } + + @Test + public void testTimeout() throws LensAPIException { + LensClientConfig config = createLensClientConfigWithServerUrl(); + + //Timeout Expected + config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS, 200); + LensClient lensClient = new LensClient(config); + assertTrue(lensClient.setDatabase(TEST_DB)); + try { + lensClient.executeQueryWithTimeout("cube select id,name from test_dim", "test1", 100000); + fail("Read Timeout was expected"); + } catch (Exception e) { + if (!(isExceptionDueToSocketTimeout(e))) { + log.error("Unexpected Exception", e); + fail("SocketTimeoutException was excepted as part of Read Timeout"); + } else { + log.debug("Expected Exception", e); + } + } + lensClient.closeConnection(); + + //No Timeout Expected + lensClient = new LensClient(config); + assertTrue(lensClient.setDatabase(TEST_DB)); + config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS, LensClientConfig.DEFAULT_READ_TIMEOUT_MILLIS); + QueryHandleWithResultSet result = lensClient.executeQueryWithTimeout("cube select id,name from test_dim", "test2", + 100000); + assertTrue(result.getStatus().successful()); + lensClient.closeConnection(); + } + + @Test + public void testWaitForQueryToCompleteWithAndWithoutRetryOnTimeOut() throws LensAPIException { + LensClientConfig config = createLensClientConfigWithServerUrl(); + config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS, 3000); + LensClient lensClient = new LensClient(config); + assertTrue(lensClient.setDatabase(TEST_DB)); + lensClient.setConnectionParam(ENABLE_SLEEP_FOR_GET_QUERY_OP, "true"); + + //Test waitForQueryToComplete without retry on timeout + QueryHandle handle = lensClient.executeQueryAsynch("cube select id,name from test_dim", "test3"); + try { + lensClient.getStatement().waitForQueryToComplete(handle, false); + fail("SocketTimeoutException was expected"); + } catch (Exception e) { + if (!isExceptionDueToSocketTimeout(e)) { + fail("SocketTimeoutException was excepted as part of Read Timeout"); + } + } + + //Test waitForQueryToComplete with Retry on timeout + handle = lensClient.executeQueryAsynch("cube select id,name from test_dim", "test3"); + lensClient.getStatement().waitForQueryToComplete(handle); + LensQuery query = lensClient.getQueryDetails(handle); + assertTrue(query.getStatus().successful()); + } + + private LensClientConfig createLensClientConfigWithServerUrl() { + LensClientConfig config = new LensClientConfig(); + config.set("lens.server.base.url", "http://localhost:" + getTestPort() + "/lensapi"); + return config; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/test/java/org/apache/lens/server/MockQueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-client/src/test/java/org/apache/lens/server/MockQueryExecutionServiceImpl.java b/lens-client/src/test/java/org/apache/lens/server/MockQueryExecutionServiceImpl.java new file mode 100644 index 0000000..9b55fb6 --- /dev/null +++ b/lens-client/src/test/java/org/apache/lens/server/MockQueryExecutionServiceImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server; + +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.query.QueryExecutionServiceImpl; + +import org.apache.hive.service.cli.CLIService; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MockQueryExecutionServiceImpl extends QueryExecutionServiceImpl { + + private static final long INITIAL_SLEEP_MILLIS = 5000; //5 secs + private static final long DECREASE_BY_MILLIS = 1000; //1 sec + private long sleepInterval = INITIAL_SLEEP_MILLIS; + public static final String ENABLE_SLEEP_FOR_GET_QUERY_OP = "test.MockQueryExecutionServiceImpl.getQuery.sleep"; + + /** + * Instantiates a new query execution service impl. + * + * @param cliService the cli service + * @throws LensException the lens exception + */ + public MockQueryExecutionServiceImpl(CLIService cliService) throws LensException { + super(cliService); + log.info("Using MockQueryExecutionServiceImpl as QueryExecutionService implementation"); + } + + @Override + public LensQuery getQuery(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException { + + if (getSession(sessionHandle).getSessionConf().get(ENABLE_SLEEP_FOR_GET_QUERY_OP) != null) { + //Introduce wait time for requests on this session. The wait time decreases with each new request to + //this method and finally becomes zero and is then reinitialized to INITIAL_SLEEP_MILLIS. + // This is used to simulate READ_TIMEOUT on client. + // Note : the logic is not synchronized as such scenario/need is not expected from test case + if (sleepInterval > 0) { + try { + log.info("MockQueryExecutionServiceImpl.getQuery Sleeping for {} millis", sleepInterval); + Thread.sleep(sleepInterval); + } catch (InterruptedException e) { + //Ignore + } + } + sleepInterval = sleepInterval - DECREASE_BY_MILLIS; + if (sleepInterval < 0) { + sleepInterval = INITIAL_SLEEP_MILLIS; + } + } + + return super.getQuery(sessionHandle, queryHandle); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index b5853bf..5466e7a 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -205,9 +205,9 @@ Lens server configuration *--+--+---+--+ |88|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|89|lens.server.status.update.delay.secs.maximum|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|89|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|90|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|90|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ |91|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/src/site/apt/user/client-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/client-config.apt b/src/site/apt/user/client-config.apt index 4ed41b4..c91c944 100644 --- a/src/site/apt/user/client-config.apt +++ b/src/site/apt/user/client-config.apt @@ -28,16 +28,20 @@ Lens client configuration *--+--+---+--+ |2|lens.cli.query.execute.timeout.millis|10000|This property defines the timeout value when sync or --async false option is used to execute a query. The default value is 10 seconds.| *--+--+---+--+ -|3|lens.client.dbname|default|Default lens database| +|3|lens.client.connection.timeout.millis|60000|This is the maximum amount of time a client is blocked for making the initial connection. The Default value for this property is 60 seconds.| *--+--+---+--+ -|4|lens.client.query.poll.interval|10000|Interval at which query progress will be polled. Interval has to be given in milliseconds| +|4|lens.client.dbname|default|Default lens database| *--+--+---+--+ -|5|lens.client.requestfilter.ws.filter.impl|org.apache.lens.client.RequestFilter|Implementation class for Request Filter| +|5|lens.client.query.poll.interval|10000|Interval at which query progress will be polled. Interval has to be given in milliseconds| *--+--+---+--+ -|6|lens.client.user.name|anonymous|Lens client user name| +|6|lens.client.read.timeout.millis|300000|This is the maximum amount of time a client read operation is blocked waiting for data. The default value of this property is 5 mins.| *--+--+---+--+ -|7|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +|7|lens.client.requestfilter.ws.filter.impl|org.apache.lens.client.RequestFilter|Implementation class for Request Filter| *--+--+---+--+ -|8|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| +|8|lens.client.user.name|anonymous|Lens client user name| +*--+--+---+--+ +|9|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +*--+--+---+--+ +|10|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| *--+--+---+--+ The configuration parameters and their default values