>From Savyasach Reddy <[email protected]>:
Savyasach Reddy has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19376 )
Change subject: [NO ISSUE]: Fix TokenAcquireTimeout and introduce Backoff for
retries
......................................................................
[NO ISSUE]: Fix TokenAcquireTimeout and introduce Backoff for retries
- user model changes: no
- storage format changes: no
- interface changes: no
details:
- Fix token acquire timeout from 5sec to 100ms
- Add a Full Jittered Exponential Backoff for retries
Ext-ref: MB-65063
Change-Id: I1c2a1ba129d725e09d7bca2ef44ecdeebe82eda0
---
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
M
hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/util/CloudRetryableRequestUtil.java
A
hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExponentialRetryPolicy.java
3 files changed, 108 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/76/19376/1
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 9f6da42..7b6fb6e 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -239,8 +239,8 @@
}
public long getTokenAcquireTimeout() {
- int time = accessor.getInt(Option.CLOUD_PROFILER_LOG_INTERVAL);
- return Math.max(time, 5000);
+ int time = accessor.getInt(Option.CLOUD_ACQUIRE_TOKEN_TIMEOUT);
+ return Math.min(time, 1000);
}
public int getWriteMaxRequestsPerSecond() {
diff --git
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/util/CloudRetryableRequestUtil.java
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/util/CloudRetryableRequestUtil.java
index 2d0afbe..6242332 100644
---
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/util/CloudRetryableRequestUtil.java
+++
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/util/CloudRetryableRequestUtil.java
@@ -25,6 +25,8 @@
import org.apache.hyracks.cloud.io.request.ICloudBeforeRetryRequest;
import org.apache.hyracks.cloud.io.request.ICloudRequest;
import org.apache.hyracks.cloud.io.request.ICloudReturnableRequest;
+import org.apache.hyracks.util.ExponentialRetryPolicy;
+import org.apache.hyracks.util.IRetryPolicy;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -39,10 +41,13 @@
* @see System#setProperty(String, String)
*/
public static final String CLOUD_UNSTABLE_MODE = "cloud.unstable.mode";
- private static final int STABLE_NUMBER_OF_RETRIES = 5;
+ private static final int STABLE_NUMBER_OF_RETRIES = 10;
+ private static final long STABLE_MAX_DELAY_BETWEEN_RETRIES = 10_000; // in
milliseconds
private static final int UNSTABLE_NUMBER_OF_RETRIES = 100;
+ private static final int UNSTABLE_MAX_DELAY_BETWEEN_RETRIES = 0;
private static final Logger LOGGER = LogManager.getLogger();
private static final int NUMBER_OF_RETRIES = getNumberOfRetries();
+ private static final long MAX_DELAY_BETWEEN_RETRIES =
getMaxDelayBetweenRetries();
private static final ICloudBeforeRetryRequest NO_OP_RETRY = () -> {
};
@@ -162,11 +167,15 @@
private static <T> T doRun(ICloudReturnableRequest<T> request,
ICloudBeforeRetryRequest retry)
throws HyracksDataException {
int attempt = 1;
+ IRetryPolicy retryPolicy = null;
while (true) {
try {
return request.call();
- } catch (IOException e) {
- if (attempt > NUMBER_OF_RETRIES) {
+ } catch (IOException | RuntimeException e) {
+ if (retryPolicy == null) {
+ retryPolicy = new
ExponentialRetryPolicy(NUMBER_OF_RETRIES, MAX_DELAY_BETWEEN_RETRIES);
+ }
+ if (!retryPolicy.retry(e)) {
throw HyracksDataException.create(e);
}
attempt++;
@@ -179,12 +188,16 @@
private static void doRun(ICloudRequest request, ICloudBeforeRetryRequest
retry) throws HyracksDataException {
int attempt = 1;
+ IRetryPolicy retryPolicy = null;
while (true) {
try {
request.call();
break;
- } catch (IOException e) {
- if (attempt > NUMBER_OF_RETRIES) {
+ } catch (IOException | RuntimeException e) {
+ if (retryPolicy == null) {
+ retryPolicy = new
ExponentialRetryPolicy(NUMBER_OF_RETRIES, MAX_DELAY_BETWEEN_RETRIES);
+ }
+ if (!retryPolicy.retry(e)) {
throw HyracksDataException.create(e);
}
attempt++;
@@ -199,4 +212,8 @@
return unstable ? UNSTABLE_NUMBER_OF_RETRIES :
STABLE_NUMBER_OF_RETRIES;
}
+ private static long getMaxDelayBetweenRetries() {
+ boolean unstable = Boolean.getBoolean(CLOUD_UNSTABLE_MODE);
+ return unstable ? UNSTABLE_MAX_DELAY_BETWEEN_RETRIES :
STABLE_MAX_DELAY_BETWEEN_RETRIES;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExponentialRetryPolicy.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExponentialRetryPolicy.java
new file mode 100644
index 0000000..ae24860
--- /dev/null
+++
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExponentialRetryPolicy.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+public class ExponentialRetryPolicy implements IRetryPolicy {
+
+ private static final int DEFAULT_MAX_RETRIES = 10;
+ private static final long DEFAULT_INITIAL_DELAY = 100; // in milliseconds
+ private static final long DEFAULT_MAX_DELAY = Long.MAX_VALUE;
+ private final int maxRetries;
+ private final long initialDelay;
+ private final long maxDelay;
+ private int attempt = 0;
+
+ public ExponentialRetryPolicy() {
+ this(DEFAULT_MAX_RETRIES, DEFAULT_INITIAL_DELAY, DEFAULT_MAX_DELAY);
+ }
+
+ public ExponentialRetryPolicy(int maxRetries, long initialDelay, long
maxDelay) {
+ this.maxRetries = maxRetries;
+ this.initialDelay = initialDelay;
+ this.maxDelay = maxDelay;
+ }
+
+ public ExponentialRetryPolicy(int maxRetries) {
+ this(maxRetries, DEFAULT_INITIAL_DELAY, DEFAULT_MAX_DELAY);
+ }
+
+ public ExponentialRetryPolicy(int maxRetries, long maxDelay) {
+ this(maxRetries, DEFAULT_INITIAL_DELAY, maxDelay);
+ }
+
+ @Override
+ public boolean retry(Throwable failure) {
+ if (attempt < maxRetries) {
+ try {
+ long delay = initialDelay * (1L << attempt);
+
TimeUnit.MILLISECONDS.sleep(ThreadLocalRandom.current().nextLong(1 +
Long.min(delay, maxDelay)));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ attempt++;
+ return true;
+ }
+ return false;
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19376
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I1c2a1ba129d725e09d7bca2ef44ecdeebe82eda0
Gerrit-Change-Number: 19376
Gerrit-PatchSet: 1
Gerrit-Owner: Savyasach Reddy <[email protected]>
Gerrit-MessageType: newchange
