[hadoop] branch branch-2.10 updated: MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-2.10 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-2.10 by this push: new 7a309f1 MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) 7a309f1 is described below commit 7a309f1f919f25b1110b7106bc1bf2f64e3814fb Author: Masatake Iwasaki AuthorDate: Wed Jan 15 12:09:09 2020 +0900 MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) (cherry picked from commit a2fdd7c2b59463c39a4d4250f908a9e2d3653ead) --- .../v2/app/speculate/forecast/SimpleExponentialSmoothing.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java index 0e00068..dd9dc28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java @@ -131,8 +131,8 @@ public class SimpleExponentialSmoothing { /** * checks if the task is hanging up. * @param timeStamp current time of the scan. - * @return true if we have number of samples > kMinimumReads and the record - * timestamp has expired. + * @return true if we have number of samples {@literal >} kMinimumReads and the + * record timestamp has expired. */ public boolean isDataStagnated(final long timeStamp) { ForecastRecord rec = forecastRefEntry.get(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new ce52a66 MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) ce52a66 is described below commit ce52a66fcbbb7c745d525cb09e9cfc03da6d9343 Author: Masatake Iwasaki AuthorDate: Wed Jan 15 12:09:09 2020 +0900 MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) (cherry picked from commit a2fdd7c2b59463c39a4d4250f908a9e2d3653ead) --- .../v2/app/speculate/forecast/SimpleExponentialSmoothing.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java index 0e00068..dd9dc28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java @@ -131,8 +131,8 @@ public class SimpleExponentialSmoothing { /** * checks if the task is hanging up. * @param timeStamp current time of the scan. - * @return true if we have number of samples > kMinimumReads and the record - * timestamp has expired. + * @return true if we have number of samples {@literal >} kMinimumReads and the + * record timestamp has expired. */ public boolean isDataStagnated(final long timeStamp) { ForecastRecord rec = forecastRefEntry.get(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (c36f09d -> a2fdd7c)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from c36f09d HADOOP-16005. NativeAzureFileSystem does not support setXAttr. add a2fdd7c MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) No new revisions were added by this update. Summary of changes: .../v2/app/speculate/forecast/SimpleExponentialSmoothing.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 483d54f MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) 483d54f is described below commit 483d54f1a712bf721623415b83dd7a71196f0bb8 Author: Masatake Iwasaki AuthorDate: Wed Jan 15 12:09:09 2020 +0900 MAPREDUCE-7256. Fix javadoc error in SimpleExponentialSmoothing. (#1804) (cherry picked from commit a2fdd7c2b59463c39a4d4250f908a9e2d3653ead) --- .../v2/app/speculate/forecast/SimpleExponentialSmoothing.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java index 0e00068..dd9dc28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/forecast/SimpleExponentialSmoothing.java @@ -131,8 +131,8 @@ public class SimpleExponentialSmoothing { /** * checks if the task is hanging up. * @param timeStamp current time of the scan. - * @return true if we have number of samples > kMinimumReads and the record - * timestamp has expired. + * @return true if we have number of samples {@literal >} kMinimumReads and the + * record timestamp has expired. */ public boolean isDataStagnated(final long timeStamp) { ForecastRecord rec = forecastRefEntry.get(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16005. NativeAzureFileSystem does not support setXAttr.
This is an automated email from the ASF dual-hosted git repository. dazhou pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new c36f09d HADOOP-16005. NativeAzureFileSystem does not support setXAttr. c36f09d is described below commit c36f09deb91454c086926c01f872d8ca4419aee0 Author: Clemens Wolff AuthorDate: Tue Jan 14 17:28:37 2020 -0800 HADOOP-16005. NativeAzureFileSystem does not support setXAttr. Contributed by Clemens Wolff. --- .../fs/azure/AzureNativeFileSystemStore.java | 67 ++--- .../hadoop/fs/azure/NativeAzureFileSystem.java | 71 ++ .../hadoop/fs/azure/NativeFileSystemStore.java | 4 + .../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 79 .../fs/azurebfs/AzureBlobFileSystemStore.java | 9 ++ .../fs/azure/NativeAzureFileSystemBaseTest.java| 62 .../ITestAzureBlobFileSystemAttributes.java| 104 + 7 files changed, 384 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 239dec2..414d2f2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -29,6 +29,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; import java.util.Calendar; import java.util.Date; @@ -247,6 +249,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private static final int DEFAULT_CONCURRENT_WRITES = 8; + private static final Charset METADATA_ENCODING = StandardCharsets.UTF_8; + // Concurrent reads reads of data written out of band are disable by default. // private static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false; @@ -1662,17 +1666,30 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { removeMetadataAttribute(blob, OLD_IS_FOLDER_METADATA_KEY); } - private static void storeLinkAttribute(CloudBlobWrapper blob, - String linkTarget) throws UnsupportedEncodingException { -// We have to URL encode the link attribute as the link URI could + private static String encodeMetadataAttribute(String value) throws UnsupportedEncodingException { +// We have to URL encode the attribute as it could // have URI special characters which unless encoded will result // in 403 errors from the server. This is due to metadata properties // being sent in the HTTP header of the request which is in turn used // on the server side to authorize the request. -String encodedLinkTarget = null; -if (linkTarget != null) { - encodedLinkTarget = URLEncoder.encode(linkTarget, "UTF-8"); -} +return value == null ? null : URLEncoder.encode(value, METADATA_ENCODING.name()); + } + + private static String decodeMetadataAttribute(String encoded) throws UnsupportedEncodingException { +return encoded == null ? null : URLDecoder.decode(encoded, METADATA_ENCODING.name()); + } + + private static String ensureValidAttributeName(String attribute) { +// Attribute names must be valid C# identifiers so we have to +// convert the namespace dots (e.g. "user.something") in the +// attribute names. Using underscores here to be consistent with +// the constant metadata keys defined earlier in the file +return attribute.replace('.', '_'); + } + + private static void storeLinkAttribute(CloudBlobWrapper blob, + String linkTarget) throws UnsupportedEncodingException { +String encodedLinkTarget = encodeMetadataAttribute(linkTarget); storeMetadataAttribute(blob, LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY, encodedLinkTarget); @@ -1686,11 +1703,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { String encodedLinkTarget = getMetadataAttribute(blob, LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY, OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY); -String linkTarget = null; -if (encodedLinkTarget != null) { - linkTarget = URLDecoder.decode(encodedLinkTarget, "UTF-8"); -} -return linkTarget; +return decodeMetadataAttribute(encodedLinkTarget); } private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) { @@ -2212,6 +2225,36 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { } @Override + public byte[] retrieveAttribute(String key, String attribute) throws
[hadoop] branch branch-2.9 updated: HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel.
This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch branch-2.9 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-2.9 by this push: new b1a7534 HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel. b1a7534 is described below commit b1a753496d01af903a1f10262acb2f336a645ce2 Author: Wei-Chiu Chuang AuthorDate: Tue Jan 14 15:39:59 2020 -0800 HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel. (cherry picked from commit c675719c3fb0fe5f6b0be624935fdf22bb228e0f) --- .../hdfs/server/datanode/checker/DatasetVolumeChecker.java | 13 - .../server/datanode/checker/TestDatasetVolumeChecker.java | 9 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index cba6710..92069f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -46,6 +46,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -103,6 +104,8 @@ public class DatasetVolumeChecker { private static final VolumeCheckContext IGNORED_CONTEXT = new VolumeCheckContext(); + private final ExecutorService checkVolumeResultHandlerExecutorService; + /** * @param conf Configuration object. * @param timer {@link Timer} object used for throttling checks. @@ -163,6 +166,12 @@ public class DatasetVolumeChecker { .setNameFormat("DataNode DiskChecker thread %d") .setDaemon(true) .build())); + +checkVolumeResultHandlerExecutorService = Executors.newCachedThreadPool( +new ThreadFactoryBuilder() +.setNameFormat("VolumeCheck ResultHandler thread %d") +.setDaemon(true) +.build()); } /** @@ -293,7 +302,9 @@ public class DatasetVolumeChecker { Futures.addCallback(olf.get(), new ResultHandler(volumeReference, new HashSet(), new HashSet(), - new AtomicLong(1), callback)); + new AtomicLong(1), callback), + checkVolumeResultHandlerExecutorService + ); return true; } else { IOUtils.cleanup(null, volumeReference); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java index 2a1c824..08aa1c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java @@ -19,11 +19,13 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import com.google.common.base.Optional; +import com.google.common.base.Supplier; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.FakeTimer; import org.junit.Rule; @@ -121,6 +123,13 @@ public class TestDatasetVolumeChecker { } }); +GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { +return (numCallbackInvocations.get() > 0); + } +}, 5, 1); + // Ensure that the check was invoked at least once. verify(volume, times(1)).check(any(VolumeCheckContext.class)); if (result) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-2 updated: HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel.
This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-2 by this push: new c675719 HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel. c675719 is described below commit c675719c3fb0fe5f6b0be624935fdf22bb228e0f Author: Wei-Chiu Chuang AuthorDate: Tue Jan 14 15:39:59 2020 -0800 HDFS-13339. Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume. Contributed by Jim Brennan, Zsolt Venczel. --- .../hdfs/server/datanode/checker/DatasetVolumeChecker.java | 13 - .../server/datanode/checker/TestDatasetVolumeChecker.java | 9 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 0f59b84..05a9ae8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -47,6 +47,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -104,6 +105,8 @@ public class DatasetVolumeChecker { private static final VolumeCheckContext IGNORED_CONTEXT = new VolumeCheckContext(); + private final ExecutorService checkVolumeResultHandlerExecutorService; + /** * @param conf Configuration object. * @param timer {@link Timer} object used for throttling checks. @@ -165,6 +168,12 @@ public class DatasetVolumeChecker { .setNameFormat("DataNode DiskChecker thread %d") .setDaemon(true) .build())); + +checkVolumeResultHandlerExecutorService = Executors.newCachedThreadPool( +new ThreadFactoryBuilder() +.setNameFormat("VolumeCheck ResultHandler thread %d") +.setDaemon(true) +.build()); } /** @@ -295,7 +304,9 @@ public class DatasetVolumeChecker { Futures.addCallback(olf.get(), new ResultHandler(volumeReference, new HashSet(), new HashSet(), - new AtomicLong(1), callback)); + new AtomicLong(1), callback), + checkVolumeResultHandlerExecutorService + ); return true; } else { IOUtils.cleanup(null, volumeReference); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java index 2a1c824..08aa1c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeChecker.java @@ -19,11 +19,13 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import com.google.common.base.Optional; +import com.google.common.base.Supplier; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.FakeTimer; import org.junit.Rule; @@ -121,6 +123,13 @@ public class TestDatasetVolumeChecker { } }); +GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { +return (numCallbackInvocations.get() > 0); + } +}, 5, 1); + // Ensure that the check was invoked at least once. verify(volume, times(1)).check(any(VolumeCheckContext.class)); if (result) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9788. Queue Management API does not support parallel updates. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 1c51f36 YARN-9788. Queue Management API does not support parallel updates. Contributed by Prabhu Joseph 1c51f36 is described below commit 1c51f36be79924489ca4a2e5ca7e96ac75a6ec18 Author: Szilard Nemeth AuthorDate: Tue Jan 14 12:26:03 2020 +0100 YARN-9788. Queue Management API does not support parallel updates. Contributed by Prabhu Joseph --- .../hadoop/yarn/client/cli/TestSchedConfCLI.java | 7 +-- .../scheduler/MutableConfigurationProvider.java| 10 ++-- .../conf/FSSchedulerConfigurationStore.java| 8 +-- .../capacity/conf/InMemoryConfigurationStore.java | 10 ++-- .../capacity/conf/LeveldbConfigurationStore.java | 6 +-- .../conf/MutableCSConfigurationProvider.java | 8 +-- .../capacity/conf/YarnConfigurationStore.java | 6 ++- .../capacity/conf/ZKConfigurationStore.java| 7 +-- .../resourcemanager/webapp/RMWebServices.java | 8 +-- .../capacity/conf/ConfigurationStoreBaseTest.java | 6 +-- .../conf/TestFSSchedulerConfigurationStore.java| 8 +-- .../conf/TestLeveldbConfigurationStore.java| 16 +++--- .../conf/TestMutableCSConfigurationProvider.java | 57 +- .../capacity/conf/TestZKConfigurationStore.java| 30 ++-- 14 files changed, 116 insertions(+), 71 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 4233b4c..3b961df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; @@ -247,10 +248,10 @@ public class TestSchedConfCLI extends JerseyTestBase { globalUpdates.put("schedKey1", "schedVal1"); schedUpdateInfo.setGlobalParams(globalUpdates); - provider.logAndApplyMutation(UserGroupInformation.getCurrentUser(), - schedUpdateInfo); + LogMutation log = provider.logAndApplyMutation( + UserGroupInformation.getCurrentUser(), schedUpdateInfo); rm.getRMContext().getRMAdminService().refreshQueues(); - provider.confirmPendingMutation(true); + provider.confirmPendingMutation(log, true); Configuration schedulerConf = provider.getConfiguration(); assertEquals("schedVal1", schedulerConf.get("schedKey1")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 03902e3..751c9a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -46,18 +47,21 @@ public interface MutableConfigurationProvider { * Log user's requested configuration mutation, and applies it in-memory. * @param user User who requested the change * @param confUpdate User's
[hadoop] branch branch-3.1 updated: HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 3a09593 HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal 3a09593 is described below commit 3a09593a73c1c919bebf344bd4a2afdddcbb85c9 Author: Szilard Nemeth AuthorDate: Tue Jan 14 11:19:52 2020 +0100 HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal --- .../org/apache/hadoop/io/retry/RetryPolicies.java | 12 +++- .../org/apache/hadoop/io/retry/TestRetryProxy.java| 19 +++ .../hadoop/io/retry/UnreliableImplementation.java | 7 +++ .../apache/hadoop/io/retry/UnreliableInterface.java | 4 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index a89c3a7..fcbcc86 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -690,7 +690,8 @@ public class RetryPolicies { } else if (e instanceof InvalidToken) { return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "Invalid or Cancelled Token"); - } else if (e instanceof AccessControlException) { + } else if (e instanceof AccessControlException || + hasWrappedAccessControlException(e)) { return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "Access denied"); } else if (e instanceof SocketException @@ -761,4 +762,13 @@ public class RetryPolicies { return unwrapped instanceof RetriableException ? (RetriableException) unwrapped : null; } + + private static boolean hasWrappedAccessControlException(Exception e) { +Throwable throwable = e; +while (!(throwable instanceof AccessControlException) && +throwable.getCause() != null) { + throwable = throwable.getCause(); +} +return throwable instanceof AccessControlException; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java index 2116fb2..a1135a0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java @@ -377,4 +377,23 @@ public class TestRetryProxy { assertEquals(RetryDecision.FAIL, caughtRetryAction.action); } } + + @Test + public void testWrappedAccessControlException() throws Exception { +RetryPolicy policy = mock(RetryPolicy.class); +RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5); +setupMockPolicy(policy, realPolicy); + +UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create( +UnreliableInterface.class, unreliableImpl, policy); + +try { + unreliable.failsWithWrappedAccessControlException(); + fail("Should fail"); +} catch (IOException expected) { + verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(), + anyInt(), anyBoolean()); + assertEquals(RetryDecision.FAIL, caughtRetryAction.action); +} + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java index a20d898..15a84bb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java @@ -139,6 +139,13 @@ class UnreliableImplementation implements UnreliableInterface { } } + public void failsWithWrappedAccessControlException() + throws IOException { +AccessControlException ace = new AccessControlException(); +IOException ioe = new IOException(ace); +throw new IOException(ioe); + } + @Override public String succeedsOnceThenFailsReturningString() throws UnreliableException, IOException, StandbyException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java index 738a760..80bf47d 100644 ---
[hadoop] branch branch-3.2 updated: HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 1d2c5df HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal 1d2c5df is described below commit 1d2c5dffa8489f389b9f52138feb67b61c492306 Author: Szilard Nemeth AuthorDate: Tue Jan 14 11:18:54 2020 +0100 HADOOP-16683. Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException. Contributed by Adam Antal --- .../org/apache/hadoop/io/retry/RetryPolicies.java | 12 +++- .../org/apache/hadoop/io/retry/TestRetryProxy.java| 19 +++ .../hadoop/io/retry/UnreliableImplementation.java | 7 +++ .../apache/hadoop/io/retry/UnreliableInterface.java | 4 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index a89c3a7..fcbcc86 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -690,7 +690,8 @@ public class RetryPolicies { } else if (e instanceof InvalidToken) { return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "Invalid or Cancelled Token"); - } else if (e instanceof AccessControlException) { + } else if (e instanceof AccessControlException || + hasWrappedAccessControlException(e)) { return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "Access denied"); } else if (e instanceof SocketException @@ -761,4 +762,13 @@ public class RetryPolicies { return unwrapped instanceof RetriableException ? (RetriableException) unwrapped : null; } + + private static boolean hasWrappedAccessControlException(Exception e) { +Throwable throwable = e; +while (!(throwable instanceof AccessControlException) && +throwable.getCause() != null) { + throwable = throwable.getCause(); +} +return throwable instanceof AccessControlException; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java index 2116fb2..a1135a0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java @@ -377,4 +377,23 @@ public class TestRetryProxy { assertEquals(RetryDecision.FAIL, caughtRetryAction.action); } } + + @Test + public void testWrappedAccessControlException() throws Exception { +RetryPolicy policy = mock(RetryPolicy.class); +RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5); +setupMockPolicy(policy, realPolicy); + +UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create( +UnreliableInterface.class, unreliableImpl, policy); + +try { + unreliable.failsWithWrappedAccessControlException(); + fail("Should fail"); +} catch (IOException expected) { + verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(), + anyInt(), anyBoolean()); + assertEquals(RetryDecision.FAIL, caughtRetryAction.action); +} + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java index a20d898..15a84bb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java @@ -139,6 +139,13 @@ class UnreliableImplementation implements UnreliableInterface { } } + public void failsWithWrappedAccessControlException() + throws IOException { +AccessControlException ace = new AccessControlException(); +IOException ioe = new IOException(ace); +throw new IOException(ioe); + } + @Override public String succeedsOnceThenFailsReturningString() throws UnreliableException, IOException, StandbyException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java index 738a760..80bf47d 100644 ---
[hadoop] branch trunk updated: YARN-10028. Integrate the new abstract log servlet to the JobHistory server. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 13cea04 YARN-10028. Integrate the new abstract log servlet to the JobHistory server. Contributed by Adam Antal 13cea04 is described below commit 13cea0412c11ce9ef7e475198a24e71788cf0b2f Author: Szilard Nemeth AuthorDate: Tue Jan 14 11:00:08 2020 +0100 YARN-10028. Integrate the new abstract log servlet to the JobHistory server. Contributed by Adam Antal --- .../mapreduce/v2/hs/HistoryClientService.java | 7 ++- .../mapreduce/v2/hs/webapp/HsWebServices.java | 58 -- .../mapreduce/v2/hs/webapp/TestHsWebServices.java | 3 ++ .../v2/hs/webapp/TestHsWebServicesAcls.java| 2 +- .../v2/hs/webapp/TestHsWebServicesAttempts.java| 3 ++ .../v2/hs/webapp/TestHsWebServicesJobConf.java | 3 ++ .../v2/hs/webapp/TestHsWebServicesJobs.java| 4 ++ .../v2/hs/webapp/TestHsWebServicesJobsQuery.java | 3 ++ .../v2/hs/webapp/TestHsWebServicesTasks.java | 3 ++ .../org/apache/hadoop/yarn/webapp/WebApps.java | 12 - .../hadoop/yarn/webapp/WebServicesTestUtils.java | 1 - .../hadoop/yarn/server/webapp/AppInfoProvider.java | 2 + .../hadoop/yarn/server/webapp/BasicAppInfo.java| 4 ++ 13 files changed, 96 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index b0bf41b..b63aef4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -79,6 +79,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -150,9 +152,11 @@ public class HistoryClientService extends AbstractService { } @VisibleForTesting - protected void initializeWebApp(Configuration conf) { + protected void initializeWebApp(Configuration conf) throws IOException { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); +ApplicationClientProtocol appClientProtocol = +ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") @@ -163,6 +167,7 @@ public class HistoryClientService extends AbstractService { JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .withCSRFProtection(JHAdminConfig.MR_HISTORY_CSRF_PREFIX) .withXFSProtection(JHAdminConfig.MR_HISTORY_XFS_PREFIX) +.withAppClientProtocol(appClientProtocol) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index dabb760..e3804e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -20,8 +20,10 @@ package org.apache.hadoop.mapreduce.v2.hs.webapp; import java.io.IOException; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -30,9 +32,12 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response;