[ https://issues.apache.org/jira/browse/HADOOP-18632?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17691470#comment-17691470 ]
ASF GitHub Bot commented on HADOOP-18632: ----------------------------------------- sreeb-msft commented on code in PR #5399: URL: https://github.com/apache/hadoop/pull/5399#discussion_r1112771893 ########## hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimeoutOptimizer.java: ########## @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams; +import org.apache.http.client.utils.URIBuilder; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; + +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DEFAULT_TIMEOUT; + +public class TimeoutOptimizer { + AbfsConfiguration abfsConfiguration; + private URL url; + private AbfsRestOperationType opType; + private ExponentialRetryPolicy retryPolicy; + private int requestTimeout; + private int readTimeout = -1; + private int connTimeout = -1; + private int maxReqTimeout; + private int timeoutIncRate; + private boolean shouldOptimizeTimeout; + + public TimeoutOptimizer(URL url, AbfsRestOperationType opType, ExponentialRetryPolicy retryPolicy, AbfsConfiguration abfsConfiguration) { + this.url = url; + this.opType = opType; + if (opType != null) { + this.retryPolicy = retryPolicy; + this.abfsConfiguration = abfsConfiguration; + if (abfsConfiguration.get(ConfigurationKeys.AZURE_OPTIMIZE_TIMEOUTS) == null) { + this.shouldOptimizeTimeout = false; + } + else { + this.shouldOptimizeTimeout = Boolean.parseBoolean(abfsConfiguration.get(ConfigurationKeys.AZURE_OPTIMIZE_TIMEOUTS)); + } + if (this.shouldOptimizeTimeout) { + this.maxReqTimeout = Integer.parseInt(abfsConfiguration.get(ConfigurationKeys.AZURE_MAX_REQUEST_TIMEOUT)); + this.timeoutIncRate = Integer.parseInt(abfsConfiguration.get(ConfigurationKeys.AZURE_REQUEST_TIMEOUT_INCREASE_RATE)); + initTimeouts(); + updateUrl(); + } + + } else { + this.shouldOptimizeTimeout = false; + } + } + + public void updateRetryTimeout(int retryCount) { + if (!this.shouldOptimizeTimeout) { + return; + } + + // update all timeout values + updateTimeouts(retryCount); + updateUrl(); + } + + public URL getUrl() { + return url; + } + public boolean getShouldOptimizeTimeout() { return this.shouldOptimizeTimeout; } + + public int getRequestTimeout() { return requestTimeout; } + + public int getReadTimeout() { + return readTimeout; + } + + public int getReadTimeout(final int defaultTimeout) { + if (readTimeout != -1 && shouldOptimizeTimeout) { + return readTimeout; + } + return defaultTimeout; + } + + public int getConnTimeout() { + return connTimeout; + } + + public int getConnTimeout(final int defaultTimeout) { + if (connTimeout == -1) { + return defaultTimeout; + } + return connTimeout; + } + + private void initTimeouts() { + if (!shouldOptimizeTimeout) { + requestTimeout = -1; + readTimeout = -1; + connTimeout = -1; + return; + } + + String query = url.getQuery(); + int timeoutPos = query.indexOf("timeout"); + if (timeoutPos < 0) { Review Comment: Understood, thanks! > ABFS: Customize and optimize timeouts made based on each separate request > ------------------------------------------------------------------------- > > Key: HADOOP-18632 > URL: https://issues.apache.org/jira/browse/HADOOP-18632 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure > Reporter: Sree Bhattacharyya > Assignee: Sree Bhattacharyya > Priority: Minor > Labels: pull-request-available > > In present day ABFS Driver functioning, all API request calls use the same > values of default timeouts. This is sub-optimal in the scenarios where a > request is failing due to hitting a particular busy node, and would benefit > simply by retrying quicker. > For this, the change to be brought in chooses customized timeouts based on > which API call is being made. Further, starting with smaller, optimized > values of timeouts, the timeout values would increase by a certain > incremental factor for subsequent retries to ensure quicker retries and > success. -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org