hadoop git commit: HDDS-353. Multiple delete Blocks tests are failing consistently. Contributed by Lokesh Jain.
Repository: hadoop Updated Branches: refs/heads/trunk 6425ed27e -> e3d73bbc2 HDDS-353. Multiple delete Blocks tests are failing consistently. Contributed by Lokesh Jain. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3d73bbc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3d73bbc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3d73bbc Branch: refs/heads/trunk Commit: e3d73bbc24eea8d539811ba07c695b1d0c139b61 Parents: 6425ed2 Author: Mukul Kumar Singh Authored: Mon Aug 20 13:37:58 2018 +0530 Committer: Mukul Kumar Singh Committed: Mon Aug 20 13:37:58 2018 +0530 -- .../apache/hadoop/hdds/scm/HddsServerUtil.java | 6 +++--- .../report/CommandStatusReportPublisher.java | 2 +- .../common/report/ContainerReportPublisher.java | 2 +- .../common/report/NodeReportPublisher.java | 2 +- .../statemachine/DatanodeStateMachine.java | 3 +-- .../statemachine/EndpointStateMachine.java | 9 + .../hdds/scm/container/ContainerMapping.java | 7 --- .../scm/container/closer/ContainerCloser.java| 9 - .../ozone/TestStorageContainerManager.java | 19 ++- .../commandhandler/TestBlockDeletion.java| 10 +++--- .../apache/hadoop/ozone/web/client/TestKeys.java | 10 +- 11 files changed, 50 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d73bbc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java -- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java index a8b919d..580d027 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java @@ -180,11 +180,11 @@ public final class HddsServerUtil { * SCM. * * @param conf - Ozone Config - * @return - HB interval in seconds. + * @return - HB interval in milli seconds. */ public static long getScmHeartbeatInterval(Configuration conf) { return conf.getTimeDuration(HDDS_HEARTBEAT_INTERVAL, -HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS); +HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); } /** @@ -202,7 +202,7 @@ public final class HddsServerUtil { long heartbeatThreadFrequencyMs = getScmheartbeatCheckerInterval(conf); -long heartbeatIntervalMs = getScmHeartbeatInterval(conf) * 1000; +long heartbeatIntervalMs = getScmHeartbeatInterval(conf); // Make sure that StaleNodeInterval is configured way above the frequency http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d73bbc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java -- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java index 3898d15..4cf6321 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java @@ -58,7 +58,7 @@ public class CommandStatusReportPublisher extends getConf()); Preconditions.checkState( - heartbeatFrequency < cmdStatusReportInterval, + heartbeatFrequency <= cmdStatusReportInterval, HDDS_COMMAND_STATUS_REPORT_INTERVAL + " cannot be configured lower than heartbeat frequency."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d73bbc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java -- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java index 3e73bb4..ccb9a9a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/repor
hadoop git commit: HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma.
Repository: hadoop Updated Branches: refs/heads/trunk e3d73bbc2 -> 01ff81781 HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01ff8178 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01ff8178 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01ff8178 Branch: refs/heads/trunk Commit: 01ff8178148790f7b0112058cf08d23d031b6868 Parents: e3d73bb Author: Yiqun Lin Authored: Mon Aug 20 23:01:59 2018 +0800 Committer: Yiqun Lin Committed: Mon Aug 20 23:01:59 2018 +0800 -- .../server/federation/router/RouterRpcClient.java | 18 ++ .../server/federation/router/RouterRpcServer.java | 3 ++- .../hdfs/server/federation/MockResolver.java | 3 +++ .../hdfs/server/federation/router/TestRouter.java | 18 ++ 4 files changed, 33 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ff8178/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 3eb7241..56ca55f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -92,8 +92,8 @@ public class RouterRpcClient { LoggerFactory.getLogger(RouterRpcClient.class); - /** Router identifier. */ - private final String routerId; + /** Router using this RPC client. */ + private final Router router; /** Interface to identify the active NN for a nameservice or blockpool ID. */ private final ActiveNamenodeResolver namenodeResolver; @@ -116,12 +116,13 @@ public class RouterRpcClient { * Create a router RPC client to manage remote procedure calls to NNs. * * @param conf Hdfs Configuation. + * @param router A router using this RPC client. * @param resolver A NN resolver to determine the currently active NN in HA. * @param monitor Optional performance monitor. */ - public RouterRpcClient(Configuration conf, String identifier, + public RouterRpcClient(Configuration conf, Router router, ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { -this.routerId = identifier; +this.router = router; this.namenodeResolver = resolver; @@ -343,7 +344,8 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + - " with params " + Arrays.toString(params) + " from " + this.routerId); + " with params " + Arrays.toString(params) + " from " + + router.getRouterId()); } Object ret = null; @@ -1126,7 +1128,7 @@ public class RouterRpcClient { String msg = "Not enough client threads " + active + "/" + total; LOG.error(msg); throw new StandbyException( - "Router " + routerId + " is overloaded: " + msg); + "Router " + router.getRouterId() + " is overloaded: " + msg); } catch (InterruptedException ex) { LOG.error("Unexpected error while invoking API: {}", ex.getMessage()); throw new IOException( @@ -1150,7 +1152,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } @@ -1171,7 +1173,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + bpId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ff8178/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index fe54993..2deda9f 100644 --- a/hadoop-hdfs-project/hado
hadoop git commit: HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 675aa2bbc -> 5e2e68d4b HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma. (cherry picked from commit 01ff8178148790f7b0112058cf08d23d031b6868) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e2e68d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e2e68d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e2e68d4 Branch: refs/heads/branch-3.0 Commit: 5e2e68d4bb3af43edc6f45d5ca0672fb1ec0d7f2 Parents: 675aa2b Author: Yiqun Lin Authored: Mon Aug 20 23:01:59 2018 +0800 Committer: Yiqun Lin Committed: Mon Aug 20 23:03:19 2018 +0800 -- .../server/federation/router/RouterRpcClient.java | 18 ++ .../server/federation/router/RouterRpcServer.java | 3 ++- .../hdfs/server/federation/MockResolver.java | 3 +++ .../hdfs/server/federation/router/TestRouter.java | 18 ++ 4 files changed, 33 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e2e68d4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 3eb7241..56ca55f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -92,8 +92,8 @@ public class RouterRpcClient { LoggerFactory.getLogger(RouterRpcClient.class); - /** Router identifier. */ - private final String routerId; + /** Router using this RPC client. */ + private final Router router; /** Interface to identify the active NN for a nameservice or blockpool ID. */ private final ActiveNamenodeResolver namenodeResolver; @@ -116,12 +116,13 @@ public class RouterRpcClient { * Create a router RPC client to manage remote procedure calls to NNs. * * @param conf Hdfs Configuation. + * @param router A router using this RPC client. * @param resolver A NN resolver to determine the currently active NN in HA. * @param monitor Optional performance monitor. */ - public RouterRpcClient(Configuration conf, String identifier, + public RouterRpcClient(Configuration conf, Router router, ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { -this.routerId = identifier; +this.router = router; this.namenodeResolver = resolver; @@ -343,7 +344,8 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + - " with params " + Arrays.toString(params) + " from " + this.routerId); + " with params " + Arrays.toString(params) + " from " + + router.getRouterId()); } Object ret = null; @@ -1126,7 +1128,7 @@ public class RouterRpcClient { String msg = "Not enough client threads " + active + "/" + total; LOG.error(msg); throw new StandbyException( - "Router " + routerId + " is overloaded: " + msg); + "Router " + router.getRouterId() + " is overloaded: " + msg); } catch (InterruptedException ex) { LOG.error("Unexpected error while invoking API: {}", ex.getMessage()); throw new IOException( @@ -1150,7 +1152,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } @@ -1171,7 +1173,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + bpId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e2e68d4/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Ro
hadoop git commit: HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 a3d4a25bb -> d7442c244 HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma. (cherry picked from commit 01ff8178148790f7b0112058cf08d23d031b6868) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7442c24 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7442c24 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7442c24 Branch: refs/heads/branch-3.1 Commit: d7442c244f5490e66e89d3a4e787c170807ce338 Parents: a3d4a25 Author: Yiqun Lin Authored: Mon Aug 20 23:01:59 2018 +0800 Committer: Yiqun Lin Committed: Mon Aug 20 23:05:55 2018 +0800 -- .../server/federation/router/RouterRpcClient.java | 18 ++ .../server/federation/router/RouterRpcServer.java | 3 ++- .../hdfs/server/federation/MockResolver.java | 3 +++ .../hdfs/server/federation/router/TestRouter.java | 18 ++ 4 files changed, 33 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7442c24/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 3eb7241..56ca55f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -92,8 +92,8 @@ public class RouterRpcClient { LoggerFactory.getLogger(RouterRpcClient.class); - /** Router identifier. */ - private final String routerId; + /** Router using this RPC client. */ + private final Router router; /** Interface to identify the active NN for a nameservice or blockpool ID. */ private final ActiveNamenodeResolver namenodeResolver; @@ -116,12 +116,13 @@ public class RouterRpcClient { * Create a router RPC client to manage remote procedure calls to NNs. * * @param conf Hdfs Configuation. + * @param router A router using this RPC client. * @param resolver A NN resolver to determine the currently active NN in HA. * @param monitor Optional performance monitor. */ - public RouterRpcClient(Configuration conf, String identifier, + public RouterRpcClient(Configuration conf, Router router, ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { -this.routerId = identifier; +this.router = router; this.namenodeResolver = resolver; @@ -343,7 +344,8 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + - " with params " + Arrays.toString(params) + " from " + this.routerId); + " with params " + Arrays.toString(params) + " from " + + router.getRouterId()); } Object ret = null; @@ -1126,7 +1128,7 @@ public class RouterRpcClient { String msg = "Not enough client threads " + active + "/" + total; LOG.error(msg); throw new StandbyException( - "Router " + routerId + " is overloaded: " + msg); + "Router " + router.getRouterId() + " is overloaded: " + msg); } catch (InterruptedException ex) { LOG.error("Unexpected error while invoking API: {}", ex.getMessage()); throw new IOException( @@ -1150,7 +1152,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } @@ -1171,7 +1173,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + bpId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7442c24/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Ro
hadoop git commit: HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma.
Repository: hadoop Updated Branches: refs/heads/branch-2 c68d1d49c -> 18ebe1830 HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18ebe183 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18ebe183 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18ebe183 Branch: refs/heads/branch-2 Commit: 18ebe183052ee37ce041432f3b32a3b9a531c3e6 Parents: c68d1d4 Author: Yiqun Lin Authored: Mon Aug 20 23:08:26 2018 +0800 Committer: Yiqun Lin Committed: Mon Aug 20 23:08:26 2018 +0800 -- .../federation/router/RouterRpcClient.java | 18 +--- .../federation/router/RouterRpcServer.java | 3 ++- .../hdfs/server/federation/MockResolver.java| 3 +++ .../server/federation/router/TestRouter.java| 22 4 files changed, 37 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/18ebe183/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index e4d304d..0161900 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -92,8 +92,8 @@ public class RouterRpcClient { LoggerFactory.getLogger(RouterRpcClient.class); - /** Router identifier. */ - private final String routerId; + /** Router using this RPC client. */ + private final Router router; /** Interface to identify the active NN for a nameservice or blockpool ID. */ private final ActiveNamenodeResolver namenodeResolver; @@ -116,12 +116,13 @@ public class RouterRpcClient { * Create a router RPC client to manage remote procedure calls to NNs. * * @param conf Hdfs Configuation. + * @param router A router using this RPC client. * @param resolver A NN resolver to determine the currently active NN in HA. * @param monitor Optional performance monitor. */ - public RouterRpcClient(Configuration conf, String identifier, + public RouterRpcClient(Configuration conf, Router router, ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { -this.routerId = identifier; +this.router = router; this.namenodeResolver = resolver; @@ -335,7 +336,8 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + - " with params " + Arrays.toString(params) + " from " + this.routerId); + " with params " + Arrays.toString(params) + " from " + + router.getRouterId()); } Object ret = null; @@ -1118,7 +1120,7 @@ public class RouterRpcClient { String msg = "Not enough client threads " + active + "/" + total; LOG.error(msg); throw new StandbyException( - "Router " + routerId + " is overloaded: " + msg); + "Router " + router.getRouterId() + " is overloaded: " + msg); } catch (InterruptedException ex) { LOG.error("Unexpected error while invoking API: {}", ex.getMessage()); throw new IOException( @@ -1142,7 +1144,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } @@ -1163,7 +1165,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + bpId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/18ebe183/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 7dff226..9c07a9a 100644 --- a/hadoop-hdfs-project/hadoop
hadoop git commit: HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma.
Repository: hadoop Updated Branches: refs/heads/branch-2.9 42c47971d -> 9c8a79806 HDFS-13750. RBF: Router ID in RouterRpcClient is always null. Contributed by Takanobu Asanuma. (cherry picked from commit 18ebe183052ee37ce041432f3b32a3b9a531c3e6) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c8a7980 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c8a7980 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c8a7980 Branch: refs/heads/branch-2.9 Commit: 9c8a79806d2218c6dfd79d373030074ea981d0c4 Parents: 42c4797 Author: Yiqun Lin Authored: Mon Aug 20 23:08:26 2018 +0800 Committer: Yiqun Lin Committed: Mon Aug 20 23:10:02 2018 +0800 -- .../federation/router/RouterRpcClient.java | 18 +--- .../federation/router/RouterRpcServer.java | 3 ++- .../hdfs/server/federation/MockResolver.java| 3 +++ .../server/federation/router/TestRouter.java| 22 4 files changed, 37 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c8a7980/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index e4d304d..0161900 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -92,8 +92,8 @@ public class RouterRpcClient { LoggerFactory.getLogger(RouterRpcClient.class); - /** Router identifier. */ - private final String routerId; + /** Router using this RPC client. */ + private final Router router; /** Interface to identify the active NN for a nameservice or blockpool ID. */ private final ActiveNamenodeResolver namenodeResolver; @@ -116,12 +116,13 @@ public class RouterRpcClient { * Create a router RPC client to manage remote procedure calls to NNs. * * @param conf Hdfs Configuation. + * @param router A router using this RPC client. * @param resolver A NN resolver to determine the currently active NN in HA. * @param monitor Optional performance monitor. */ - public RouterRpcClient(Configuration conf, String identifier, + public RouterRpcClient(Configuration conf, Router router, ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { -this.routerId = identifier; +this.router = router; this.namenodeResolver = resolver; @@ -335,7 +336,8 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + - " with params " + Arrays.toString(params) + " from " + this.routerId); + " with params " + Arrays.toString(params) + " from " + + router.getRouterId()); } Object ret = null; @@ -1118,7 +1120,7 @@ public class RouterRpcClient { String msg = "Not enough client threads " + active + "/" + total; LOG.error(msg); throw new StandbyException( - "Router " + routerId + " is overloaded: " + msg); + "Router " + router.getRouterId() + " is overloaded: " + msg); } catch (InterruptedException ex) { LOG.error("Unexpected error while invoking API: {}", ex.getMessage()); throw new IOException( @@ -1142,7 +1144,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } @@ -1163,7 +1165,7 @@ public class RouterRpcClient { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("Cannot locate a registered namenode for " + bpId + - " from " + this.routerId); + " from " + router.getRouterId()); } return namenodes; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c8a7980/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcS
hadoop git commit: YARN-8242. YARN NM: OOM error while reading back the state store on recovery. Contributed by Pradeep Ambati and Kanwaljeet Sachdev
Repository: hadoop Updated Branches: refs/heads/trunk 01ff81781 -> 65e746971 YARN-8242. YARN NM: OOM error while reading back the state store on recovery. Contributed by Pradeep Ambati and Kanwaljeet Sachdev Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65e74697 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65e74697 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65e74697 Branch: refs/heads/trunk Commit: 65e7469712be6cf393e29ef73cc94727eec81227 Parents: 01ff817 Author: Jason Lowe Authored: Mon Aug 20 10:14:40 2018 -0500 Committer: Jason Lowe Committed: Mon Aug 20 10:14:40 2018 -0500 -- .../server/nodemanager/DeletionService.java | 25 +- .../containermanager/ContainerManagerImpl.java | 26 +- .../localizer/ResourceLocalizationService.java | 56 +-- .../recovery/NMLeveldbStateStoreService.java| 412 --- .../recovery/NMNullStateStoreService.java | 2 +- .../recovery/NMStateStoreService.java | 55 +-- .../nodemanager/recovery/RecoveryIterator.java | 41 ++ .../security/NMContainerTokenSecretManager.java | 27 +- .../security/NMTokenSecretManagerInNM.java | 15 +- .../recovery/NMMemoryStateStoreService.java | 82 +++- .../TestNMLeveldbStateStoreService.java | 216 +++--- 11 files changed, 647 insertions(+), 310 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/65e74697/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java index ae81dc1..e665c5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java @@ -19,13 +19,14 @@ package org.apache.hadoop.yarn.server.nodemanager; import static java.util.concurrent.TimeUnit.SECONDS; + +import org.apache.hadoop.yarn.server.nodemanager.recovery.RecoveryIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -96,16 +97,20 @@ public class DeletionService extends AbstractService { private void recover(NMStateStoreService.RecoveredDeletionServiceState state) throws IOException { -List taskProtos = state.getTasks(); Map idToInfoMap = -new HashMap<>(taskProtos.size()); -Set successorTasks = new HashSet<>(); -for (DeletionServiceDeleteTaskProto proto : taskProtos) { - DeletionTaskRecoveryInfo info = - NMProtoUtils.convertProtoToDeletionTaskRecoveryInfo(proto, this); - idToInfoMap.put(info.getTask().getTaskId(), info); - nextTaskId.set(Math.max(nextTaskId.get(), info.getTask().getTaskId())); - successorTasks.addAll(info.getSuccessorTaskIds()); +new HashMap(); +Set successorTasks = new HashSet(); + +try (RecoveryIterator it = + state.getIterator()) { + while (it.hasNext()) { +DeletionServiceDeleteTaskProto proto = it.next(); +DeletionTaskRecoveryInfo info = +NMProtoUtils.convertProtoToDeletionTaskRecoveryInfo(proto, this); +idToInfoMap.put(info.getTask().getTaskId(), info); +nextTaskId.set(Math.max(nextTaskId.get(), info.getTask().getTaskId())); +successorTasks.addAll(info.getSuccessorTaskIds()); + } } // restore the task dependencies and schedule the deletion tasks that http://git-wip-us.apache.org/repos/asf/hadoop/blob/65e74697/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/ha
hadoop git commit: YARN-8242. YARN NM: OOM error while reading back the state store on recovery. Contributed by Pradeep Ambati and Kanwaljeet Sachdev
Repository: hadoop Updated Branches: refs/heads/branch-3.1 d7442c244 -> 44c4928b6 YARN-8242. YARN NM: OOM error while reading back the state store on recovery. Contributed by Pradeep Ambati and Kanwaljeet Sachdev (cherry picked from commit 65e7469712be6cf393e29ef73cc94727eec81227) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44c4928b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44c4928b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44c4928b Branch: refs/heads/branch-3.1 Commit: 44c4928b64498c76fc5dffe288d9e959960282df Parents: d7442c2 Author: Jason Lowe Authored: Mon Aug 20 10:14:40 2018 -0500 Committer: Jason Lowe Committed: Mon Aug 20 10:21:57 2018 -0500 -- .../server/nodemanager/DeletionService.java | 25 +- .../containermanager/ContainerManagerImpl.java | 26 +- .../localizer/ResourceLocalizationService.java | 56 +-- .../recovery/NMLeveldbStateStoreService.java| 412 --- .../recovery/NMNullStateStoreService.java | 2 +- .../recovery/NMStateStoreService.java | 55 +-- .../nodemanager/recovery/RecoveryIterator.java | 41 ++ .../security/NMContainerTokenSecretManager.java | 27 +- .../security/NMTokenSecretManagerInNM.java | 15 +- .../recovery/NMMemoryStateStoreService.java | 82 +++- .../TestNMLeveldbStateStoreService.java | 216 +++--- 11 files changed, 647 insertions(+), 310 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c4928b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java index ae81dc1..e665c5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java @@ -19,13 +19,14 @@ package org.apache.hadoop.yarn.server.nodemanager; import static java.util.concurrent.TimeUnit.SECONDS; + +import org.apache.hadoop.yarn.server.nodemanager.recovery.RecoveryIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -96,16 +97,20 @@ public class DeletionService extends AbstractService { private void recover(NMStateStoreService.RecoveredDeletionServiceState state) throws IOException { -List taskProtos = state.getTasks(); Map idToInfoMap = -new HashMap<>(taskProtos.size()); -Set successorTasks = new HashSet<>(); -for (DeletionServiceDeleteTaskProto proto : taskProtos) { - DeletionTaskRecoveryInfo info = - NMProtoUtils.convertProtoToDeletionTaskRecoveryInfo(proto, this); - idToInfoMap.put(info.getTask().getTaskId(), info); - nextTaskId.set(Math.max(nextTaskId.get(), info.getTask().getTaskId())); - successorTasks.addAll(info.getSuccessorTaskIds()); +new HashMap(); +Set successorTasks = new HashSet(); + +try (RecoveryIterator it = + state.getIterator()) { + while (it.hasNext()) { +DeletionServiceDeleteTaskProto proto = it.next(); +DeletionTaskRecoveryInfo info = +NMProtoUtils.convertProtoToDeletionTaskRecoveryInfo(proto, this); +idToInfoMap.put(info.getTask().getTaskId(), info); +nextTaskId.set(Math.max(nextTaskId.get(), info.getTask().getTaskId())); +successorTasks.addAll(info.getSuccessorTaskIds()); + } } // restore the task dependencies and schedule the deletion tasks that http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c4928b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-ya
hadoop git commit: YARN-8673. [AMRMProxy] More robust responseId resync after an YarnRM master slave switch. Contributed by Botong Huang.
Repository: hadoop Updated Branches: refs/heads/trunk 65e746971 -> 8736fc39a YARN-8673. [AMRMProxy] More robust responseId resync after an YarnRM master slave switch. Contributed by Botong Huang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8736fc39 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8736fc39 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8736fc39 Branch: refs/heads/trunk Commit: 8736fc39ac3b3de168d2c216f3d1c0edb48fb3f9 Parents: 65e7469 Author: Giovanni Matteo Fumarola Authored: Mon Aug 20 12:22:36 2018 -0700 Committer: Giovanni Matteo Fumarola Committed: Mon Aug 20 12:22:36 2018 -0700 -- .../hadoop/yarn/client/AMRMClientUtils.java | 47 +++ .../hadoop/yarn/server/AMRMClientRelayer.java | 130 +-- .../server/uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 53 +++- .../amrmproxy/FederationInterceptor.java| 6 +- .../ApplicationMasterService.java | 9 +- 6 files changed, 190 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8736fc39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java index 5d4ab4a6..b8319cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java @@ -36,6 +36,7 @@ import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -54,6 +55,10 @@ public final class AMRMClientUtils { public static final String APP_ALREADY_REGISTERED_MESSAGE = "Application Master is already registered : "; + public static final String EXPECTED_HB_RESPONSEID_MESSAGE = + " expect responseId to be "; + public static final String RECEIVED_HB_RESPONSEID_MESSAGE = " but get "; + private AMRMClientUtils() { } @@ -105,6 +110,48 @@ public final class AMRMClientUtils { SaslRpcServer.AuthMethod.TOKEN.toString()); } + /** + * Generate the exception message when RM receives an AM heartbeat with + * invalid responseId. + * + * @param appAttemptId the app attempt + * @param expected the expected responseId value + * @param received the received responseId value + * @return the assembled exception message + */ + public static String assembleInvalidResponseIdExceptionMessage( + ApplicationAttemptId appAttemptId, int expected, int received) { +return "Invalid responseId in AllocateRequest from application attempt: " ++ appAttemptId + EXPECTED_HB_RESPONSEID_MESSAGE + expected ++ RECEIVED_HB_RESPONSEID_MESSAGE + received; + } + + /** + * Parse the expected responseId from the exception generated by RM when + * processing AM heartbeat. + * + * @param exceptionMessage the exception message thrown by RM + * @return the parsed expected responseId, -1 if failed + */ + public static int parseExpectedResponseIdFromException( + String exceptionMessage) { +if (exceptionMessage == null) { + return -1; +} +int start = exceptionMessage.indexOf(EXPECTED_HB_RESPONSEID_MESSAGE); +int end = exceptionMessage.indexOf(RECEIVED_HB_RESPONSEID_MESSAGE); +if (start == -1 || end == -1) { + return -1; +} +start += EXPECTED_HB_RESPONSEID_MESSAGE.length(); + +try { + return Integer.parseInt(exceptionMessage.substring(start, end)); +} catch (NumberFormatException ex) { + return -1; +} + } + public static void addToOutstandingSchedulingRequests( Collection requests, Map, List> outstandingSchedRequests) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8736fc39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/had
hadoop git commit: YARN-8581. [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy. Contributed by Botong Huang.
Repository: hadoop Updated Branches: refs/heads/trunk 8736fc39a -> e0f6ffdba YARN-8581. [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy. Contributed by Botong Huang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0f6ffdb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0f6ffdb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0f6ffdb Branch: refs/heads/trunk Commit: e0f6ffdbad6f43fd43ec57fb68ebf5275b8b9ba0 Parents: 8736fc3 Author: Giovanni Matteo Fumarola Authored: Mon Aug 20 14:33:16 2018 -0700 Committer: Giovanni Matteo Fumarola Committed: Mon Aug 20 14:33:16 2018 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 8 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../LocalityMulticastAMRMProxyPolicy.java | 64 -- .../utils/FederationStateStoreFacade.java | 9 ++ .../TestLocalityMulticastAMRMProxyPolicy.java | 91 +--- .../utils/FederationPoliciesTestUtil.java | 7 +- 6 files changed, 162 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0f6ffdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 78e28f7..148edb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3209,8 +3209,14 @@ public class YarnConfiguration extends Configuration { "org.apache.hadoop.yarn.server.federation.resolver." + "DefaultSubClusterResolverImpl"; - public static final String DEFAULT_FEDERATION_POLICY_KEY = "*"; + // AMRMProxy split-merge timeout for active sub-clusters. We will not route + // new asks to expired sub-clusters. + public static final String FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT = + FEDERATION_PREFIX + "amrmproxy.subcluster.timeout.ms"; + public static final long DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT = + 6; // one minute + public static final String DEFAULT_FEDERATION_POLICY_KEY = "*"; public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX + "policy-manager"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0f6ffdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9249ed4..d63933c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -105,6 +105,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER); configurationPropsToSkipCompare .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS); +configurationPropsToSkipCompare +.add(YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT); // Federation StateStore ZK implementation configs to be ignored configurationPropsToSkipCompare.add( http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0f6ffdb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java index 1481f34..1ccd61c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn
[04/14] hadoop git commit: HDFS-13799. TestEditLogTailer#testTriggersLogRollsForAllStandbyNN fails due to missing synchronization between rollEditsRpcExecutor and tailerThread shutdown. Contributed by
HDFS-13799. TestEditLogTailer#testTriggersLogRollsForAllStandbyNN fails due to missing synchronization between rollEditsRpcExecutor and tailerThread shutdown. Contributed by Hrishikesh Gadre. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75406990 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75406990 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75406990 Branch: refs/heads/branch-3.1 Commit: 754069906bda1d961bce9f584d0457ee10db6762 Parents: e4b75ad Author: Xiao Chen Authored: Tue Aug 7 16:11:37 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:51:07 2018 -0700 -- .../org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/75406990/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index 2003f94..b306b8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -234,7 +234,6 @@ public class EditLogTailer { } public void stop() throws IOException { -rollEditsRpcExecutor.shutdown(); tailerThread.setShouldRun(false); tailerThread.interrupt(); try { @@ -242,6 +241,8 @@ public class EditLogTailer { } catch (InterruptedException e) { LOG.warn("Edit log tailer thread exited with an exception"); throw new IOException(e); +} finally { + rollEditsRpcExecutor.shutdown(); } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[10/14] hadoop git commit: HDFS-13217. Audit log all EC policy names during addErasureCodingPolicies. Contributed by liaoyuxiangqin.
HDFS-13217. Audit log all EC policy names during addErasureCodingPolicies. Contributed by liaoyuxiangqin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81fbfe50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81fbfe50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81fbfe50 Branch: refs/heads/branch-3.1 Commit: 81fbfe500e4ac1c57c681db20f620336a072898e Parents: c15853f Author: Xiao Chen Authored: Wed Aug 15 09:22:24 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:56:47 2018 -0700 -- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 10 ++ 1 file changed, 6 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81fbfe50/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 5eef12b..f6c8c94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7475,9 +7475,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ErasureCodingPolicy[] policies, final boolean logRetryCache) throws IOException { final String operationName = "addErasureCodingPolicies"; -String addECPolicyName = ""; +List addECPolicyNames = new ArrayList<>(policies.length); checkOperation(OperationCategory.WRITE); -List responses = new ArrayList<>(); +List responses = +new ArrayList<>(policies.length); boolean success = false; writeLock(); try { @@ -7488,7 +7489,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ErasureCodingPolicy newPolicy = FSDirErasureCodingOp.addErasureCodingPolicy(this, policy, logRetryCache); - addECPolicyName = newPolicy.getName(); + addECPolicyNames.add(newPolicy.getName()); responses.add(new AddErasureCodingPolicyResponse(newPolicy)); } catch (HadoopIllegalArgumentException e) { responses.add(new AddErasureCodingPolicyResponse(policy, e)); @@ -7501,7 +7502,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (success) { getEditLog().logSync(); } - logAuditEvent(success, operationName, addECPolicyName, null, null); + logAuditEvent(success, operationName, addECPolicyNames.toString(), + null, null); } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[06/14] hadoop git commit: HDFS-13447. Fix Typos - Node Not Chosen. Contributed by Beluga Behr.
HDFS-13447. Fix Typos - Node Not Chosen. Contributed by Beluga Behr. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/366517ba Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/366517ba Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/366517ba Branch: refs/heads/branch-3.1 Commit: 366517ba994c9a075d220a03ae6b829e66449b55 Parents: a517ee4 Author: Márton Elek Authored: Wed Aug 8 17:27:57 2018 +0200 Committer: Arpit Agarwal Committed: Mon Aug 20 14:52:04 2018 -0700 -- .../hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/366517ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index e2a9c55..ac20e6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -67,11 +67,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { .withInitial(() -> new HashMap()); private enum NodeNotChosenReason { -NOT_IN_SERVICE("the node isn't in service"), +NOT_IN_SERVICE("the node is not in service"), NODE_STALE("the node is stale"), NODE_TOO_BUSY("the node is too busy"), TOO_MANY_NODES_ON_RACK("the rack has too many chosen nodes"), -NOT_ENOUGH_STORAGE_SPACE("no enough storage space to place the block"); +NOT_ENOUGH_STORAGE_SPACE("not enough storage space to place the block"); private final String text; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[14/14] hadoop git commit: HADOOP-9214. Create a new touch command to allow modifying atime and mtime. Contributed by Hrishikesh Gadre.
HADOOP-9214. Create a new touch command to allow modifying atime and mtime. Contributed by Hrishikesh Gadre. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3712b79b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3712b79b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3712b79b Branch: refs/heads/branch-3.1 Commit: 3712b79b38754f5e1710d29ffc3bb3576bacf02e Parents: a630a27 Author: Xiao Chen Authored: Fri Aug 17 10:53:22 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:59:14 2018 -0700 -- .../org/apache/hadoop/fs/shell/FsCommand.java | 2 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 85 .../apache/hadoop/fs/shell/TouchCommands.java | 198 +++ .../src/site/markdown/FileSystemShell.md| 32 +++ .../org/apache/hadoop/fs/TestFsShellTouch.java | 103 ++ .../src/test/resources/testConf.xml | 51 + 6 files changed, 385 insertions(+), 86 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3712b79b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java index 4a13414..784bbf3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -66,7 +66,7 @@ abstract public class FsCommand extends Command { factory.registerCommands(Tail.class); factory.registerCommands(Head.class); factory.registerCommands(Test.class); -factory.registerCommands(Touch.class); +factory.registerCommands(TouchCommands.class); factory.registerCommands(Truncate.class); factory.registerCommands(SnapshotCommands.class); factory.registerCommands(XAttrCommands.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3712b79b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java deleted file mode 100644 index a6c751e..000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touch.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.shell; - -import java.io.IOException; -import java.util.LinkedList; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.PathIOException; -import org.apache.hadoop.fs.PathIsDirectoryException; -import org.apache.hadoop.fs.PathNotFoundException; - -/** - * Unix touch like commands - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable - -class Touch extends FsCommand { - public static void registerCommands(CommandFactory factory) { -factory.addClass(Touchz.class, "-touchz"); - } - - /** - * (Re)create zero-length file at the specified path. - * This will be replaced by a more UNIX-like touch when files may be - * modified. - */ - public static class Touchz extends Touch { -public static final String NAME = "touchz"; -public static final String USAGE = " ..."; -public static final String DESCRIPTION = - "Creates a file of zero length " + - "at with current time as the timestamp of that . " + - "An error is returned if the file exists with non-zero length\n"; - -@Override -protected void processOptions(LinkedList args) { - CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE); - cf.parse
[12/14] hadoop git commit: HDFS-10240. Race between close/recoverLease leads to missing block. Contributed by Jinglun, zhouyingchao and Wei-Chiu Chuang.
HDFS-10240. Race between close/recoverLease leads to missing block. Contributed by Jinglun, zhouyingchao and Wei-Chiu Chuang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86565005 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86565005 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86565005 Branch: refs/heads/branch-3.1 Commit: 865650052b07c8a20d51306202354ac770ed36d5 Parents: 0424715 Author: Wei-Chiu Chuang Authored: Thu Aug 16 16:29:38 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:57:40 2018 -0700 -- .../hdfs/server/blockmanagement/BlockInfo.java | 4 ++ .../server/blockmanagement/BlockManager.java| 4 ++ .../hdfs/server/datanode/BPServiceActor.java| 3 +- .../hadoop/hdfs/server/datanode/DataNode.java | 10 +++ .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 65 .../hdfs/server/datanode/DataNodeTestUtils.java | 3 + 6 files changed, 88 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86565005/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 111ade1..43f4f47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -262,6 +262,10 @@ public abstract class BlockInfo extends Block return getBlockUCState().equals(BlockUCState.COMPLETE); } + public boolean isUnderRecovery() { +return getBlockUCState().equals(BlockUCState.UNDER_RECOVERY); + } + public final boolean isCompleteOrCommitted() { final BlockUCState state = getBlockUCState(); return state.equals(BlockUCState.COMPLETE) || http://git-wip-us.apache.org/repos/asf/hadoop/blob/86565005/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index bac89bf..6ab237f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -971,6 +971,10 @@ public class BlockManager implements BlockStatsMXBean { return false; // no blocks in file yet if(lastBlock.isComplete()) return false; // already completed (e.g. by syncBlock) +if(lastBlock.isUnderRecovery()) { + throw new IOException("Commit or complete block " + commitBlock + + ", whereas it is under recovery."); +} final boolean committed = commitBlock(lastBlock, commitBlock); if (committed && lastBlock.isStriped()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/86565005/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index a94d2df..6c167f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -682,7 +682,8 @@ class BPServiceActor implements Runnable { } } } -if (ibrManager.sendImmediately() || sendHeartbeat) { +if (!dn.areIBRDisabledForTests() && +(ibrManager.sendImmediately()|| sendHeartbeat)) { ibrManager.sendIBRs(bpNamenode, bpRegistration, bpos.getBlockPoolId()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86565005/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b
[09/14] hadoop git commit: HDFS-13819. TestDirectoryScanner#testDirectoryScannerInFederatedCluster is flaky
HDFS-13819. TestDirectoryScanner#testDirectoryScannerInFederatedCluster is flaky Change-Id: I1cea6e67fcec72702ad202775dee3373261ac5cd Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c15853f8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c15853f8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c15853f8 Branch: refs/heads/branch-3.1 Commit: c15853f87ae1ce4a474494f88407786b3a644068 Parents: 975d606 Author: Daniel Templeton Authored: Tue Aug 14 17:03:10 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:54:58 2018 -0700 -- .../server/datanode/TestDirectoryScanner.java | 42 +++- 1 file changed, 32 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c15853f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index f792523..893fe20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -40,6 +40,7 @@ import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.FileUtils; @@ -312,18 +313,29 @@ public class TestDirectoryScanner { return id; } - private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile, - long missingMemoryBlocks, long mismatchBlocks) throws IOException { + private void scan(long totalBlocks, int diffsize, long missingMetaFile, + long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks) + throws IOException, InterruptedException, TimeoutException { scan(totalBlocks, diffsize, missingMetaFile, missingBlockFile, missingMemoryBlocks, mismatchBlocks, 0); } private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks, - long duplicateBlocks) throws IOException { + long duplicateBlocks) + throws IOException, InterruptedException, TimeoutException { scanner.reconcile(); -verifyStats(totalBlocks, diffsize, missingMetaFile, missingBlockFile, -missingMemoryBlocks, mismatchBlocks, duplicateBlocks); + +GenericTestUtils.waitFor(() -> { + try { +verifyStats(totalBlocks, diffsize, missingMetaFile, missingBlockFile, +missingMemoryBlocks, mismatchBlocks, duplicateBlocks); + } catch (AssertionError ex) { +return false; + } + + return true; +}, 50, 2000); } private void verifyStats(long totalBlocks, int diffsize, long missingMetaFile, @@ -785,7 +797,8 @@ public class TestDirectoryScanner { } } - private float runThrottleTest(int blocks) throws IOException { + private float runThrottleTest(int blocks) + throws IOException, InterruptedException, TimeoutException { scanner.setRetainDiffs(true); scan(blocks, 0, 0, 0, 0, 0); scanner.shutdown(); @@ -1069,10 +1082,19 @@ public class TestDirectoryScanner { scanner.setRetainDiffs(true); scanner.reconcile(); //Check blocks in corresponding BP - bpid = cluster.getNamesystem(1).getBlockPoolId(); - verifyStats(bp1Files, 0, 0, 0, 0, 0, 0); - bpid = cluster.getNamesystem(3).getBlockPoolId(); - verifyStats(bp2Files, 0, 0, 0, 0, 0, 0); + + GenericTestUtils.waitFor(() -> { +try { + bpid = cluster.getNamesystem(1).getBlockPoolId(); + verifyStats(bp1Files, 0, 0, 0, 0, 0, 0); + bpid = cluster.getNamesystem(3).getBlockPoolId(); + verifyStats(bp2Files, 0, 0, 0, 0, 0, 0); +} catch (AssertionError ex) { + return false; +} + +return true; + }, 50, 2000); } finally { if (scanner != null) { scanner.shutdown(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[08/14] hadoop git commit: HDFS-13668. FSPermissionChecker may throws AIOOE when check inode permission. Contributed by He Xiaoqiao.
HDFS-13668. FSPermissionChecker may throws AIOOE when check inode permission. Contributed by He Xiaoqiao. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/975d6068 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/975d6068 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/975d6068 Branch: refs/heads/branch-3.1 Commit: 975d60685eaf9961bdbd3547600b3e38bb088835 Parents: c0ac0a5 Author: drankye Authored: Mon Aug 13 17:32:56 2018 +0800 Committer: Arpit Agarwal Committed: Mon Aug 20 14:53:32 2018 -0700 -- .../server/namenode/FSPermissionChecker.java| 2 +- .../namenode/TestINodeAttributeProvider.java| 43 ++-- 2 files changed, 41 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/975d6068/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java index 354b4e3..f70963c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java @@ -409,7 +409,7 @@ public class FSPermissionChecker implements AccessControlEnforcer { } final FsPermission mode = inode.getFsPermission(); final AclFeature aclFeature = inode.getAclFeature(); -if (aclFeature != null) { +if (aclFeature != null && aclFeature.getEntriesSize() > 0) { // It's possible that the inode has a default ACL but no access ACL. int firstEntry = aclFeature.getEntryAt(0); if (AclEntryStatusFormat.getScope(firstEntry) == AclEntryScope.ACCESS) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/975d6068/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java index 9c7dcd3..b3bab06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java @@ -57,6 +57,11 @@ public class TestINodeAttributeProvider { public static class MyAuthorizationProvider extends INodeAttributeProvider { public static class MyAccessControlEnforcer implements AccessControlEnforcer { + AccessControlEnforcer ace; + + public MyAccessControlEnforcer(AccessControlEnforcer defaultEnforcer) { +this.ace = defaultEnforcer; + } @Override public void checkPermission(String fsOwner, String supergroup, @@ -65,6 +70,13 @@ public class TestINodeAttributeProvider { int ancestorIndex, boolean doCheckOwner, FsAction ancestorAccess, FsAction parentAccess, FsAction access, FsAction subAccess, boolean ignoreEmptyDir) throws AccessControlException { +if (ancestorIndex > 1 +&& inodes[1].getLocalName().equals("user") +&& inodes[2].getLocalName().equals("acl")) { + this.ace.checkPermission(fsOwner, supergroup, ugi, inodeAttrs, inodes, + pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, + ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); +} CALLED.add("checkPermission|" + ancestorAccess + "|" + parentAccess + "|" + access); } } @@ -84,6 +96,7 @@ public class TestINodeAttributeProvider { final INodeAttributes inode) { CALLED.add("getAttributes"); final boolean useDefault = useDefault(pathElements); + final boolean useNullAcl = useNullAclFeature(pathElements); return new INodeAttributes() { @Override public boolean isDirectory() { @@ -126,7 +139,10 @@ public class TestINodeAttributeProvider { @Override public AclFeature getAclFeature() { AclFeature f; - if (useDefault) { + if (useNullAcl) { +int[] entries = new int[0]; +f = new AclFeature(entries); + } else if (useDefault) { f = inode.getAclFeature(); } else { AclEntry acl = new
[07/14] hadoop git commit: HDFS-13658. Expose HighestPriorityLowRedundancy blocks statistics. Contributed by Kitti Nanasi.
HDFS-13658. Expose HighestPriorityLowRedundancy blocks statistics. Contributed by Kitti Nanasi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ac0a53 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ac0a53 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ac0a53 Branch: refs/heads/branch-3.1 Commit: c0ac0a533701a56b09b302a660c3233971512168 Parents: 366517b Author: Xiao Chen Authored: Wed Aug 8 10:36:44 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:52:28 2018 -0700 -- .../hadoop-common/src/site/markdown/Metrics.md | 2 + .../hadoop/hdfs/protocol/ECBlockGroupStats.java | 27 +++- .../hdfs/protocol/ReplicatedBlockStats.java | 28 - .../hadoop/hdfs/protocolPB/PBHelperClient.java | 21 ++ .../src/main/proto/ClientNamenodeProtocol.proto | 3 ++ .../federation/metrics/NamenodeBeanMetrics.java | 10 + .../server/federation/router/ErasureCoding.java | 13 ++ .../server/blockmanagement/BlockManager.java| 8 .../blockmanagement/LowRedundancyBlocks.java| 28 + .../hdfs/server/namenode/FSNamesystem.java | 20 - .../hdfs/server/namenode/NameNodeMXBean.java| 18 .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 10 + .../TestLowRedundancyBlockQueues.java | 43 +--- .../namenode/metrics/TestNameNodeMetrics.java | 12 ++ .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 32 +++ 15 files changed, 247 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ac0a53/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index f1dbc50..4d59c6e 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -240,6 +240,8 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat | | `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) | | `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 | +| `HighestPriorityLowRedundancyReplicatedBlocks` | Current number of non-corrupt, low redundancy replicated blocks with the highest risk of loss (have 0 or 1 replica). Will be recovered with the highest priority. | +| `HighestPriorityLowRedundancyECBlocks` | Current number of non-corrupt, low redundancy EC blocks with the highest risk of loss. Will be recovered with the highest priority. | | `NumFilesUnderConstruction` | Current number of files under construction | | `NumActiveClients` | Current number of active clients holding lease | | `HAState` | (HA-only) Current state of the NameNode: initializing or active or standby or stopping state | http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ac0a53/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java index 9a8ad8c..3dde604 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java @@ -34,15 +34,26 @@ public final class ECBlockGroupStats { private final long missingBlockGroups; private final long bytesInFutureBlockGroups; private final long pendingDeletionBlocks; + private final Long highestPriorityLowRedundancyBlocks; public ECBlockGroupStats(long lowRedundancyBlockGroups, long corruptBlockGroups, long missingBlockGroups, long bytesInFutureBlockGroups, long pendingDeletionBlocks) { +this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups, +bytesInFutureBlockGroups, pendingDeletionBlocks, null); + } + + public ECBlockGroupStats(long lowRedundancyBlockGroups, + long corruptBlockGroups, long missingBlockGroups, + long bytesInFutureBlockGroups, long pendingDeletionBlocks, + Long highestPriorityLowRedundancyBlocks) { this.lowRedundancyBlockGroups = lowRedundancyBlockGroups; this.corruptBlockGroups = corruptBlockGrou
[13/14] hadoop git commit: HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi.
HADOOP-15655. Enhance KMS client retry behavior. Contributed by Kitti Nanasi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a630a27c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a630a27c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a630a27c Branch: refs/heads/branch-3.1 Commit: a630a27c53107322a72f9b76e395c4537b09c3fc Parents: 8656500 Author: Xiao Chen Authored: Thu Aug 16 22:32:32 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:57:51 2018 -0700 -- .../key/kms/LoadBalancingKMSClientProvider.java | 43 ++--- .../kms/TestLoadBalancingKMSClientProvider.java | 181 ++- 2 files changed, 193 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a630a27c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java index 9677b0d..e0ffdb1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java @@ -113,8 +113,8 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements return providers; } - private T doOp(ProviderCallable op, int currPos) - throws IOException { + private T doOp(ProviderCallable op, int currPos, + boolean isIdempotent) throws IOException { if (providers.length == 0) { throw new IOException("No providers configured !"); } @@ -143,7 +143,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements } RetryAction action = null; try { - action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false); + action = retryPolicy.shouldRetry(ioe, 0, numFailovers, isIdempotent); } catch (Exception e) { if (e instanceof IOException) { throw (IOException)e; @@ -201,7 +201,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements public Token[] call(KMSClientProvider provider) throws IOException { return provider.addDelegationTokens(renewer, credentials); } -}, nextIdx()); +}, nextIdx(), false); } @Override @@ -211,7 +211,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements public Long call(KMSClientProvider provider) throws IOException { return provider.renewDelegationToken(token); } -}, nextIdx()); +}, nextIdx(), false); } @Override @@ -222,7 +222,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements provider.cancelDelegationToken(token); return null; } -}, nextIdx()); +}, nextIdx(), false); } // This request is sent to all providers in the load-balancing group @@ -275,7 +275,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements throws IOException, GeneralSecurityException { return provider.generateEncryptedKey(encryptionKeyName); } - }, nextIdx()); + }, nextIdx(), true); } catch (WrapperException we) { if (we.getCause() instanceof GeneralSecurityException) { throw (GeneralSecurityException) we.getCause(); @@ -295,7 +295,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements throws IOException, GeneralSecurityException { return provider.decryptEncryptedKey(encryptedKeyVersion); } - }, nextIdx()); + }, nextIdx(), true); } catch (WrapperException we) { if (we.getCause() instanceof GeneralSecurityException) { throw (GeneralSecurityException) we.getCause(); @@ -315,7 +315,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements throws IOException, GeneralSecurityException { return provider.reencryptEncryptedKey(ekv); } - }, nextIdx()); + }, nextIdx(), true); } catch (WrapperException we) { if (we.getCause() instanceof GeneralSecurityException) { throw (GeneralSecurityException) we.getCause(); @@ -335,7 +335,7 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements provider.reencryptEncryptedKeys(ekvs); return null; } - }, nextIdx()); + }, nextIdx(), true); } catch
[03/14] hadoop git commit: HDFS-13792. Fix FSN read/write lock metrics name. Contributed by Chao Sun.
HDFS-13792. Fix FSN read/write lock metrics name. Contributed by Chao Sun. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4b75ad5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4b75ad5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4b75ad5 Branch: refs/heads/branch-3.1 Commit: e4b75ad583fe7d78a529288992f3721f29213f89 Parents: 0d155de Author: Yiqun Lin Authored: Tue Aug 7 09:32:51 2018 +0800 Committer: Arpit Agarwal Committed: Mon Aug 20 14:50:47 2018 -0700 -- .../hadoop-common/src/site/markdown/Metrics.md | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4b75ad5/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 676ab0b..f1dbc50 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -252,8 +252,10 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `NumInMaintenanceLiveDataNodes` | Number of live Datanodes which are in maintenance state | | `NumInMaintenanceDeadDataNodes` | Number of dead Datanodes which are in maintenance state | | `NumEnteringMaintenanceDataNodes` | Number of Datanodes that are entering the maintenance state | -| `FSN(Read/Write)Lock`*OperationName*`NumOps` | Total number of acquiring lock by operations | -| `FSN(Read/Write)Lock`*OperationName*`AvgTime` | Average time of holding the lock by operations in milliseconds | +| `FSN(Read/Write)Lock`*OperationName*`NanosNumOps` | Total number of acquiring lock by operations | +| `FSN(Read/Write)Lock`*OperationName*`NanosAvgTime` | Average time of holding the lock by operations in nanoseconds | +| `FSN(Read/Write)LockOverallNanosNumOps` | Total number of acquiring lock by all operations | +| `FSN(Read/Write)LockOverallNanosAvgTime` | Average time of holding the lock by all operations in nanoseconds | JournalNode --- - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[01/14] hadoop git commit: HADOOP-14212. Expose SecurityEnabled boolean field in JMX for other services besides NameNode. Contributed by Adam Antal.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 44c4928b6 -> 3712b79b3 HADOOP-14212. Expose SecurityEnabled boolean field in JMX for other services besides NameNode. Contributed by Adam Antal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78fb14ba Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78fb14ba Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78fb14ba Branch: refs/heads/branch-3.1 Commit: 78fb14ba4908cc9222bc5c1239f63f7c203564a0 Parents: 44c4928 Author: Wei-Chiu Chuang Authored: Tue Aug 14 17:19:00 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:49:24 2018 -0700 -- .../hadoop/hdfs/server/datanode/DataNode.java | 5 ++ .../hdfs/server/datanode/DataNodeMXBean.java| 7 +++ .../hdfs/server/namenode/SecondaryNameNode.java | 5 ++ .../namenode/SecondaryNameNodeInfoMXBean.java | 7 +++ .../server/datanode/TestDataNodeMXBean.java | 47 +- .../server/namenode/TestSecureNameNode.java | 52 +++- .../yarn/server/nodemanager/NodeManager.java| 20 +++- .../server/resourcemanager/ResourceManager.java | 18 ++- 8 files changed, 156 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/78fb14ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 1e9c57a..4823358 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -3142,6 +3142,11 @@ public class DataNode extends ReconfigurableBase } } + @Override + public boolean isSecurityEnabled() { +return UserGroupInformation.isSecurityEnabled(); + } + public void refreshNamenodes(Configuration conf) throws IOException { blockPoolManager.refreshNamenodes(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/78fb14ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java index b5f0cd0..9d11e14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java @@ -146,4 +146,11 @@ public interface DataNodeMXBean { * @return list of slow disks */ String getSlowDisks(); + + /** + * Gets if security is enabled. + * + * @return true, if security is enabled. + */ + boolean isSecurityEnabled(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/78fb14ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index ff83e34..4d7b747 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -722,6 +722,11 @@ public class SecondaryNameNode implements Runnable, return NetUtils.getHostPortString(nameNodeAddr); } + @Override + public boolean isSecurityEnabled() { +return UserGroupInformation.isSecurityEnabled(); + } + @Override // SecondaryNameNodeInfoMXBean public long getStartTime() { return starttime; http://git-wip-us.apache.org/repos/asf/hadoop/blob/78fb14ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNodeInfoMXBean.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNodeInfoMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNodeInfoMXBean.java index 785c5ee..a042dc2
[05/14] hadoop git commit: HDFS-13785. EC: 'removePolicy' is not working for built-in/system Erasure Code policies. Contributed by Ayush Saxena
HDFS-13785. EC: 'removePolicy' is not working for built-in/system Erasure Code policies. Contributed by Ayush Saxena Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a517ee4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a517ee4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a517ee4f Branch: refs/heads/branch-3.1 Commit: a517ee4f5be39828bf18795462457fdf1bb0db00 Parents: 7540699 Author: Vinayakumar B Authored: Wed Aug 8 12:42:20 2018 +0530 Committer: Arpit Agarwal Committed: Mon Aug 20 14:51:53 2018 -0700 -- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 4 ++-- .../src/site/markdown/HDFSErasureCoding.md | 4 ++-- .../test/resources/testErasureCodingConf.xml| 22 +++- 3 files changed, 25 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a517ee4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 9b9fe14..56706b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -154,7 +154,7 @@ public class ECAdmin extends Configured implements Tool { listing.addRow("", "The path of the xml file which defines the EC policies to add"); return getShortUsage() + "\n" + - "Add a list of erasure coding policies.\n" + + "Add a list of user defined erasure coding policies.\n" + listing.toString(); } @@ -268,7 +268,7 @@ public class ECAdmin extends Configured implements Tool { TableListing listing = AdminHelper.getOptionDescriptionListing(); listing.addRow("", "The name of the erasure coding policy"); return getShortUsage() + "\n" + - "Remove an erasure coding policy.\n" + + "Remove an user defined erasure coding policy.\n" + listing.toString(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a517ee4f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index f3b920f..2e8cbbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -203,7 +203,7 @@ Below are the details about each command. * `[-addPolicies -policyFile ]` - Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and the added policy ID is in range of 64 to 127. Adding policy will fail if there are already 64 policies added. + Add a list of user defined erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and the added policy ID is in range of 64 to 127. Adding policy will fail if there are already 64 policies added. * `[-listCodecs]` @@ -211,7 +211,7 @@ Below are the details about each command. * `[-removePolicy -policy ]` - Remove an erasure coding policy. + Remove an user defined erasure coding policy. * `[-enablePolicy -policy ]` http://git-wip-us.apache.org/repos/asf/hadoop/blob/a517ee4f/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 2f7a6a7..9070367 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -154,7 +154,7 @@ SubstringComparator - Add a list of erasure coding policies + Add a list of user defined erasure coding policies SubstringComparator @@ -164,6
[11/14] hadoop git commit: HDFS-13732. ECAdmin should print the policy name when an EC policy is set. Contributed by Zsolt Venczel.
HDFS-13732. ECAdmin should print the policy name when an EC policy is set. Contributed by Zsolt Venczel. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04247152 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04247152 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04247152 Branch: refs/heads/branch-3.1 Commit: 0424715207cd07debeee5c624973e9db90d36fb6 Parents: 81fbfe5 Author: Xiao Chen Authored: Wed Aug 15 13:51:14 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:57:07 2018 -0700 -- .../main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java | 12 ++-- .../src/test/resources/testErasureCodingConf.xml| 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/04247152/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 56706b2..56d453b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -357,16 +357,16 @@ public class ECAdmin extends Configured implements Tool { final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); try { dfs.setErasureCodingPolicy(p, ecPolicyName); -if (ecPolicyName == null){ - ecPolicyName = "default"; -} -System.out.println("Set " + ecPolicyName + " erasure coding policy on" + -" " + path); + +String actualECPolicyName = dfs.getErasureCodingPolicy(p).getName(); + +System.out.println("Set " + actualECPolicyName + +" erasure coding policy on "+ path); RemoteIterator dirIt = dfs.listStatusIterator(p); if (dirIt.hasNext()) { System.out.println("Warning: setting erasure coding policy on a " + "non-empty directory will not automatically convert existing " + - "files to " + ecPolicyName + " erasure coding policy"); + "files to " + actualECPolicyName + " erasure coding policy"); } } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/04247152/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 9070367..b47d50f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -710,7 +710,7 @@ SubstringComparator - Set default erasure coding policy on /ecdir + Set RS-6-3-1024k erasure coding policy on /ecdir @@ -728,11 +728,11 @@ SubstringComparator - Set default erasure coding policy on /ecdir + Set RS-6-3-1024k erasure coding policy on /ecdir SubstringComparator - Warning: setting erasure coding policy on a non-empty directory will not automatically convert existing files to default erasure coding policy + Warning: setting erasure coding policy on a non-empty directory will not automatically convert existing files to RS-6-3-1024k erasure coding policy - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[02/14] hadoop git commit: HADOOP-14212. Addendum patch: Expose SecurityEnabled boolean field in JMX for other services besides NameNode. Contributed by Adam Antal.
HADOOP-14212. Addendum patch: Expose SecurityEnabled boolean field in JMX for other services besides NameNode. Contributed by Adam Antal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d155de1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d155de1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d155de1 Branch: refs/heads/branch-3.1 Commit: 0d155de1591781ab374803ad79c12cf1e91f1692 Parents: 78fb14b Author: Wei-Chiu Chuang Authored: Tue Aug 14 18:24:32 2018 -0700 Committer: Arpit Agarwal Committed: Mon Aug 20 14:49:28 2018 -0700 -- .../server/nodemanager/NodeManagerMXBean.java | 38 + .../nodemanager/TestNodeManagerMXBean.java | 56 .../resourcemanager/ResourceManagerMXBean.java | 38 + .../TestResourceManagerMXBean.java | 56 4 files changed, 188 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d155de1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerMXBean.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerMXBean.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerMXBean.java new file mode 100644 index 000..b4ab0aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerMXBean.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This is the JMX management interface for NodeManager. + * End users shouldn't be implementing these interfaces, and instead + * access this information through the JMX APIs. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface NodeManagerMXBean { + /** + * Gets if security is enabled. + * + * @return true, if security is enabled. + * */ + boolean isSecurityEnabled(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d155de1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerMXBean.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerMXBean.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerMXBean.java new file mode 100644 index 000..80b915c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerMXBean.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impli
[1/2] hadoop git commit: HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 3712b79b3 -> 95fcdc043 refs/heads/trunk e0f6ffdba -> 34577d2c2 HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34577d2c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34577d2c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34577d2c Branch: refs/heads/trunk Commit: 34577d2c21096046861d2deefdbb2638b411c687 Parents: e0f6ffd Author: Steve Loughran Authored: Mon Aug 20 18:36:24 2018 -0700 Committer: Steve Loughran Committed: Mon Aug 20 18:36:24 2018 -0700 -- .../fs/CommonConfigurationKeysPublic.java | 9 + .../apache/hadoop/util/ShutdownHookManager.java | 169 -- .../src/main/resources/core-default.xml | 16 + .../hadoop/util/TestShutdownHookManager.java| 328 +++ 4 files changed, 418 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/34577d2c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index c7f32f9..b101b3b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -905,5 +905,14 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_TAGS_SYSTEM = "hadoop.tags.system"; public static final String HADOOP_TAGS_CUSTOM = "hadoop.tags.custom"; + + /** Configuration option for the shutdown hook manager shutdown time: + * {@value}. */ + public static final String SERVICE_SHUTDOWN_TIMEOUT = + "hadoop.service.shutdown.timeout"; + + /** Default shutdown hook timeout: {@value} seconds. */ + public static final long SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = 30; + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/34577d2c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java index 153f92b..2ca8e55 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java @@ -17,11 +17,17 @@ */ package org.apache.hadoop.util; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -34,6 +40,9 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT_DEFAULT; + /** * The ShutdownHookManager enables running shutdownHook * in a deterministic order, higher priority first. @@ -42,53 +51,55 @@ import java.util.concurrent.atomic.AtomicBoolean; * This class registers a single JVM shutdownHook and run all the * shutdownHooks registered to it (to this class) in order based on their * priority. + * + * Unless a hook was registered with a shutdown explicitly set through + * {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + * the shutdown time allocated to it is set by the configuration option + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + * {@code core-site.xml}, with a default value of + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + * seconds. */ -public class ShutdownHookManager { +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class ShutdownHo
[2/2] hadoop git commit: HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran.
HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran. Change-Id: Ifd4a6e3f796b4dc88e97f63066289e2534c77a29 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95fcdc04 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95fcdc04 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95fcdc04 Branch: refs/heads/branch-3.1 Commit: 95fcdc04355920697337db1136ffe463d4794616 Parents: 3712b79 Author: Steve Loughran Authored: Mon Aug 20 18:44:14 2018 -0700 Committer: Steve Loughran Committed: Mon Aug 20 18:48:49 2018 -0700 -- .../fs/CommonConfigurationKeysPublic.java | 9 + .../apache/hadoop/util/ShutdownHookManager.java | 169 -- .../src/main/resources/core-default.xml | 16 + .../hadoop/util/TestShutdownHookManager.java| 328 +++ 4 files changed, 418 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/95fcdc04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 132c9bf..7a91993 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -890,5 +890,14 @@ public class CommonConfigurationKeysPublic { HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS); public static final String HADOOP_SYSTEM_TAGS = "hadoop.system.tags"; public static final String HADOOP_CUSTOM_TAGS = "hadoop.custom.tags"; + + /** Configuration option for the shutdown hook manager shutdown time: + * {@value}. */ + public static final String SERVICE_SHUTDOWN_TIMEOUT = + "hadoop.service.shutdown.timeout"; + + /** Default shutdown hook timeout: {@value} seconds. */ + public static final long SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = 30; + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/95fcdc04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java index 153f92b..2ca8e55 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java @@ -17,11 +17,17 @@ */ package org.apache.hadoop.util; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -34,6 +40,9 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT_DEFAULT; + /** * The ShutdownHookManager enables running shutdownHook * in a deterministic order, higher priority first. @@ -42,53 +51,55 @@ import java.util.concurrent.atomic.AtomicBoolean; * This class registers a single JVM shutdownHook and run all the * shutdownHooks registered to it (to this class) in order based on their * priority. + * + * Unless a hook was registered with a shutdown explicitly set through + * {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + * the shutdown time allocated to it is set by the configuration option + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + * {@code core-site.xml}, with a default value of + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + * seconds. */ -public class ShutdownHookManager { +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class ShutdownHookManager { pri
hadoop git commit: HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 5e2e68d4b -> 53ce37ad2 HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended. Contributed by Steve Loughran. Change-Id: Ifd4a6e3f796b4dc88e97f63066289e2534c77a29 (cherry picked from commit 95fcdc04355920697337db1136ffe463d4794616) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53ce37ad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53ce37ad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53ce37ad Branch: refs/heads/branch-3.0 Commit: 53ce37ad29b4fe83a5e73a4fd685348e8d13e762 Parents: 5e2e68d Author: Steve Loughran Authored: Mon Aug 20 19:18:37 2018 -0700 Committer: Steve Loughran Committed: Mon Aug 20 19:18:37 2018 -0700 -- .../fs/CommonConfigurationKeysPublic.java | 9 + .../apache/hadoop/util/ShutdownHookManager.java | 169 -- .../src/main/resources/core-default.xml | 16 + .../hadoop/util/TestShutdownHookManager.java| 328 +++ 4 files changed, 418 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ce37ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 63803c9..30d2f31 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -888,5 +888,14 @@ public class CommonConfigurationKeysPublic { "credential$", "oauth.*token$", HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS); + + /** Configuration option for the shutdown hook manager shutdown time: + * {@value}. */ + public static final String SERVICE_SHUTDOWN_TIMEOUT = + "hadoop.service.shutdown.timeout"; + + /** Default shutdown hook timeout: {@value} seconds. */ + public static final long SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = 30; + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ce37ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java index 153f92b..2ca8e55 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java @@ -17,11 +17,17 @@ */ package org.apache.hadoop.util; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -34,6 +40,9 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT_DEFAULT; + /** * The ShutdownHookManager enables running shutdownHook * in a deterministic order, higher priority first. @@ -42,53 +51,55 @@ import java.util.concurrent.atomic.AtomicBoolean; * This class registers a single JVM shutdownHook and run all the * shutdownHooks registered to it (to this class) in order based on their * priority. + * + * Unless a hook was registered with a shutdown explicitly set through + * {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + * the shutdown time allocated to it is set by the configuration option + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + * {@code core-site.xml}, with a default value of + * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + * seconds. */ -public class ShutdownHookManager { +@InterfaceAudience.Public +@InterfaceStabil
[2/2] hadoop git commit: HDFS-13772. Erasure coding: Unnecessary NameNode Logs displaying for Enabling/Disabling Erasure coding policies which are already enabled/disabled. Contributed by Ayush Saxena
HDFS-13772. Erasure coding: Unnecessary NameNode Logs displaying for Enabling/Disabling Erasure coding policies which are already enabled/disabled. Contributed by Ayush Saxena Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df2eb81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df2eb81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df2eb81 Branch: refs/heads/branch-3.1 Commit: 8df2eb8119188b8e5515295523afc23046e1db81 Parents: 95fcdc0 Author: Vinayakumar B Authored: Tue Aug 21 09:33:19 2018 +0530 Committer: Vinayakumar B Committed: Tue Aug 21 09:34:57 2018 +0530 -- .../namenode/ErasureCodingPolicyManager.java| 15 +++ .../server/namenode/FSDirErasureCodingOp.java | 22 - .../hdfs/server/namenode/FSNamesystem.java | 26 +++- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 +++--- .../server/namenode/TestNamenodeRetryCache.java | 2 +- .../namenode/ha/TestRetryCacheWithHA.java | 2 +- 6 files changed, 47 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df2eb81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 3a310da..e7de05b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -356,7 +356,7 @@ public final class ErasureCodingPolicyManager { /** * Disable an erasure coding policy by policyName. */ - public synchronized void disablePolicy(String name) { + public synchronized boolean disablePolicy(String name) { ErasureCodingPolicyInfo info = policiesByName.get(name); if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + @@ -367,27 +367,32 @@ public final class ErasureCodingPolicyManager { enabledPoliciesByName.remove(name); enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); + info.setState(ErasureCodingPolicyState.DISABLED); + LOG.info("Disable the erasure coding policy " + name); + return true; } -info.setState(ErasureCodingPolicyState.DISABLED); -LOG.info("Disable the erasure coding policy " + name); +return false; } /** * Enable an erasure coding policy by policyName. */ - public synchronized void enablePolicy(String name) { + public synchronized boolean enablePolicy(String name) { final ErasureCodingPolicyInfo info = policiesByName.get(name); if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + name + " does not exist"); } - +if (enabledPoliciesByName.containsKey(name)) { + return false; +} final ErasureCodingPolicy ecPolicy = info.getPolicy(); enabledPoliciesByName.put(name, ecPolicy); info.setState(ErasureCodingPolicyState.ENABLED); enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); LOG.info("Enable the erasure coding policy " + name); +return true; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df2eb81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 3a32db4..f6a4093 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -252,11 +252,16 @@ final class FSDirErasureCodingOp { * rebuilding * @throws IOException */ - static void enableErasureCodingPolicy(final FSNamesystem fsn, + static boolean enableErasureCodingPolicy(final FSNamesystem fsn, String ecPolicyName, final boolean logRetryCache) throws IOException { Preconditions.checkNotNull(ecPolicyName); -fsn.getErasureCodingPolicyManager().enablePolicy(ecPolicyName); -fsn.getEditLog().lo
[1/2] hadoop git commit: HDFS-13772. Erasure coding: Unnecessary NameNode Logs displaying for Enabling/Disabling Erasure coding policies which are already enabled/disabled. Contributed by Ayush Saxena
Repository: hadoop Updated Branches: refs/heads/branch-3.1 95fcdc043 -> 8df2eb811 refs/heads/trunk 34577d2c2 -> 770d9d9bb HDFS-13772. Erasure coding: Unnecessary NameNode Logs displaying for Enabling/Disabling Erasure coding policies which are already enabled/disabled. Contributed by Ayush Saxena Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/770d9d9b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/770d9d9b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/770d9d9b Branch: refs/heads/trunk Commit: 770d9d9bb686bacef601ec8c4f884dc5afa9d4e2 Parents: 34577d2 Author: Vinayakumar B Authored: Tue Aug 21 09:33:19 2018 +0530 Committer: Vinayakumar B Committed: Tue Aug 21 09:33:19 2018 +0530 -- .../namenode/ErasureCodingPolicyManager.java| 15 +++ .../server/namenode/FSDirErasureCodingOp.java | 22 - .../hdfs/server/namenode/FSNamesystem.java | 26 +++- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 +++--- .../server/namenode/TestNamenodeRetryCache.java | 2 +- .../namenode/ha/TestRetryCacheWithHA.java | 2 +- 6 files changed, 47 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/770d9d9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 3a310da..e7de05b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -356,7 +356,7 @@ public final class ErasureCodingPolicyManager { /** * Disable an erasure coding policy by policyName. */ - public synchronized void disablePolicy(String name) { + public synchronized boolean disablePolicy(String name) { ErasureCodingPolicyInfo info = policiesByName.get(name); if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + @@ -367,27 +367,32 @@ public final class ErasureCodingPolicyManager { enabledPoliciesByName.remove(name); enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); + info.setState(ErasureCodingPolicyState.DISABLED); + LOG.info("Disable the erasure coding policy " + name); + return true; } -info.setState(ErasureCodingPolicyState.DISABLED); -LOG.info("Disable the erasure coding policy " + name); +return false; } /** * Enable an erasure coding policy by policyName. */ - public synchronized void enablePolicy(String name) { + public synchronized boolean enablePolicy(String name) { final ErasureCodingPolicyInfo info = policiesByName.get(name); if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + name + " does not exist"); } - +if (enabledPoliciesByName.containsKey(name)) { + return false; +} final ErasureCodingPolicy ecPolicy = info.getPolicy(); enabledPoliciesByName.put(name, ecPolicy); info.setState(ErasureCodingPolicyState.ENABLED); enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); LOG.info("Enable the erasure coding policy " + name); +return true; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/770d9d9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 769c137..2ba840e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -253,11 +253,16 @@ final class FSDirErasureCodingOp { * rebuilding * @throws IOException */ - static void enableErasureCodingPolicy(final FSNamesystem fsn, + static boolean enableErasureCodingPolicy(final FSNamesystem fsn, String ecPolicyName, final boolean logRetryCache) throws IOException { Precondition
hadoop git commit: YARN-8129. Improve error message for invalid value in fields attribute. Contributed by Abhishek Modi.
Repository: hadoop Updated Branches: refs/heads/trunk 770d9d9bb -> d3fef7a5c YARN-8129. Improve error message for invalid value in fields attribute. Contributed by Abhishek Modi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3fef7a5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3fef7a5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3fef7a5 Branch: refs/heads/trunk Commit: d3fef7a5c5b83d27e87b5e49928254a7d1b935e5 Parents: 770d9d9 Author: Rohith Sharma K S Authored: Tue Aug 21 11:58:07 2018 +0530 Committer: Rohith Sharma K S Committed: Tue Aug 21 11:58:07 2018 +0530 -- .../timelineservice/reader/TimelineReaderWebServicesUtils.java | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fef7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java index efaecd2..63529a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java @@ -214,7 +214,11 @@ public final class TimelineReaderWebServicesUtils { String[] strs = str.split(delimiter); EnumSet fieldList = EnumSet.noneOf(Field.class); for (String s : strs) { - fieldList.add(Field.valueOf(s.trim().toUpperCase())); + try { +fieldList.add(Field.valueOf(s.trim().toUpperCase())); + } catch (IllegalArgumentException e) { +throw new IllegalArgumentException(s + " is not a valid field."); + } } return fieldList; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org