[ https://issues.apache.org/jira/browse/HADOOP-18632?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17693150#comment-17693150 ]
ASF GitHub Bot commented on HADOOP-18632: ----------------------------------------- sreeb-msft commented on code in PR #5399: URL: https://github.com/apache/hadoop/pull/5399#discussion_r1116849374 ########## hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimeoutOptimizer.java: ########## @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.http.client.utils.URIBuilder; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; + +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DEFAULT_TIMEOUT; + +/** + * Class handling whether timeout values should be optimized. + * Timeout values optimized per request level, + * based on configs in the settings. + */ +public class TimeoutOptimizer { + private AbfsConfiguration abfsConfiguration; + private URL url; + private AbfsRestOperationType opType; + private ExponentialRetryPolicy retryPolicy; + private int requestTimeout; + private int readTimeout = -1; + private int maxReqTimeout = -1; + private int timeoutIncRate = -1; + private boolean shouldOptimizeTimeout; + + /** + * Constructor to initialize the parameters in class, + * depending upon what is configured in the settings. + * @param url request URL + * @param opType operation type + * @param retryPolicy retry policy set for this instance of AbfsClient + * @param abfsConfiguration current configuration + */ + public TimeoutOptimizer(URL url, AbfsRestOperationType opType, ExponentialRetryPolicy retryPolicy, AbfsConfiguration abfsConfiguration) { + this.url = url; + this.opType = opType; + if (opType != null) { + this.retryPolicy = retryPolicy; + this.abfsConfiguration = abfsConfiguration; + String shouldOptimize = abfsConfiguration.get(ConfigurationKeys.AZURE_OPTIMIZE_TIMEOUTS); + if (shouldOptimize == null || shouldOptimize.isEmpty()) { + // config is not set + this.shouldOptimizeTimeout = false; + } + else { + this.shouldOptimizeTimeout = Boolean.parseBoolean(shouldOptimize); + if (this.shouldOptimizeTimeout) { + // config is set to true + if (abfsConfiguration.get(ConfigurationKeys.AZURE_MAX_REQUEST_TIMEOUT) != null) { + this.maxReqTimeout = Integer.parseInt(abfsConfiguration.get(ConfigurationKeys.AZURE_MAX_REQUEST_TIMEOUT)); + } + if (abfsConfiguration.get(ConfigurationKeys.AZURE_REQUEST_TIMEOUT_INCREASE_RATE) != null) { + this.timeoutIncRate = Integer.parseInt(abfsConfiguration.get(ConfigurationKeys.AZURE_REQUEST_TIMEOUT_INCREASE_RATE)); + } + if (this.maxReqTimeout == -1 || this.timeoutIncRate == -1) { + this.shouldOptimizeTimeout = false; + } else { + initTimeouts(); + updateUrl(); + } + } + } + } else { + // optimization not required for opType == null + this.shouldOptimizeTimeout = false; + } + } + + public void updateRetryTimeout(int retryCount) { + if (!this.shouldOptimizeTimeout) { + return; + } + + // update all timeout values + updateTimeouts(retryCount); + updateUrl(); + } + + public URL getUrl() { + return url; + } + public boolean getShouldOptimizeTimeout() { + return this.shouldOptimizeTimeout; + } + + public int getRequestTimeout() { + return requestTimeout; + } + + public int getReadTimeout() { + return readTimeout; + } + + public int getReadTimeout(final int defaultTimeout) { + if (readTimeout != -1 && shouldOptimizeTimeout) { + return readTimeout; + } + return defaultTimeout; + } + + private void initTimeouts() { + if (!shouldOptimizeTimeout) { + requestTimeout = -1; + readTimeout = -1; + return; + } + + String query = url.getQuery(); + Integer timeoutPos = new Integer(query.indexOf("timeout")); + if (timeoutPos != null && timeoutPos < 0) { + // no value of timeout exists in the URL + // no optimization is needed for this particular request as well + requestTimeout = -1; Review Comment: updated this accordingly, removing any init to -1. > ABFS: Customize and optimize timeouts made based on each separate request > ------------------------------------------------------------------------- > > Key: HADOOP-18632 > URL: https://issues.apache.org/jira/browse/HADOOP-18632 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure > Reporter: Sree Bhattacharyya > Assignee: Sree Bhattacharyya > Priority: Minor > Labels: pull-request-available > > In present day ABFS Driver functioning, all API request calls use the same > values of default timeouts. This is sub-optimal in the scenarios where a > request is failing due to hitting a particular busy node, and would benefit > simply by retrying quicker. > For this, the change to be brought in chooses customized timeouts based on > which API call is being made. Further, starting with smaller, optimized > values of timeouts, the timeout values would increase by a certain > incremental factor for subsequent retries to ensure quicker retries and > success. -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org