[35/50] [abbrv] hadoop git commit: HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R.
HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/941180d8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/941180d8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/941180d8 Branch: refs/heads/HDFS-10285 Commit: 941180d82fe076ab3f37c1948d129e1f314713e4 Parents: 49cad26 Author: Uma Maheswara Rao G Authored: Tue Jan 23 16:19:46 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Thu Jul 19 22:48:56 2018 +0530 -- .../hdfs/server/balancer/NameNodeConnector.java | 8 + .../hdfs/server/common/sps/BlockDispatcher.java | 186 + .../sps/BlockMovementAttemptFinished.java | 80 ++ .../server/common/sps/BlockMovementStatus.java | 53 .../common/sps/BlockStorageMovementTracker.java | 184 + .../sps/BlocksMovementsStatusHandler.java | 95 +++ .../hdfs/server/common/sps/package-info.java| 27 ++ .../datanode/BlockStorageMovementTracker.java | 186 - .../datanode/StoragePolicySatisfyWorker.java| 271 ++- .../hdfs/server/namenode/FSNamesystem.java | 4 +- .../namenode/sps/BlockMoveTaskHandler.java | 3 +- .../sps/BlockStorageMovementAttemptedItems.java | 12 +- .../IntraSPSNameNodeBlockMoveTaskHandler.java | 3 +- .../hdfs/server/namenode/sps/SPSService.java| 14 +- .../namenode/sps/StoragePolicySatisfier.java| 30 +- .../sps/ExternalSPSBlockMoveTaskHandler.java| 233 .../TestBlockStorageMovementAttemptedItems.java | 2 +- .../sps/TestStoragePolicySatisfier.java | 6 +- .../sps/TestExternalStoragePolicySatisfier.java | 69 - 19 files changed, 997 insertions(+), 469 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/941180d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index b0dd779..6bfbbb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -269,6 +269,14 @@ public class NameNodeConnector implements Closeable { } } + /** + * Returns fallbackToSimpleAuth. This will be true or false during calls to + * indicate if a secure client falls back to simple auth. + */ + public AtomicBoolean getFallbackToSimpleAuth() { +return fallbackToSimpleAuth; + } + @Override public void close() { keyManager.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/941180d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java new file mode 100644 index 000..f87fcae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common.sps; + +import static org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import
[35/50] [abbrv] hadoop git commit: HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R.
HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c97485c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c97485c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c97485c Branch: refs/heads/HDFS-10285 Commit: 0c97485ca4a115508c3b0ee080570af16af8666b Parents: 88e087d Author: Uma Maheswara Rao G Authored: Tue Jan 23 16:19:46 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Sun Jul 15 20:22:03 2018 +0530 -- .../hdfs/server/balancer/NameNodeConnector.java | 8 + .../hdfs/server/common/sps/BlockDispatcher.java | 186 + .../sps/BlockMovementAttemptFinished.java | 80 ++ .../server/common/sps/BlockMovementStatus.java | 53 .../common/sps/BlockStorageMovementTracker.java | 184 + .../sps/BlocksMovementsStatusHandler.java | 95 +++ .../hdfs/server/common/sps/package-info.java| 27 ++ .../datanode/BlockStorageMovementTracker.java | 186 - .../datanode/StoragePolicySatisfyWorker.java| 271 ++- .../hdfs/server/namenode/FSNamesystem.java | 4 +- .../namenode/sps/BlockMoveTaskHandler.java | 3 +- .../sps/BlockStorageMovementAttemptedItems.java | 12 +- .../IntraSPSNameNodeBlockMoveTaskHandler.java | 3 +- .../hdfs/server/namenode/sps/SPSService.java| 14 +- .../namenode/sps/StoragePolicySatisfier.java| 30 +- .../sps/ExternalSPSBlockMoveTaskHandler.java| 233 .../TestBlockStorageMovementAttemptedItems.java | 2 +- .../sps/TestStoragePolicySatisfier.java | 6 +- .../sps/TestExternalStoragePolicySatisfier.java | 69 - 19 files changed, 997 insertions(+), 469 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c97485c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index b0dd779..6bfbbb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -269,6 +269,14 @@ public class NameNodeConnector implements Closeable { } } + /** + * Returns fallbackToSimpleAuth. This will be true or false during calls to + * indicate if a secure client falls back to simple auth. + */ + public AtomicBoolean getFallbackToSimpleAuth() { +return fallbackToSimpleAuth; + } + @Override public void close() { keyManager.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c97485c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java new file mode 100644 index 000..f87fcae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common.sps; + +import static org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import
[35/50] [abbrv] hadoop git commit: HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R.
HDFS-13033: [SPS]: Implement a mechanism to do file block movements for external SPS. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b527527 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b527527 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b527527 Branch: refs/heads/HDFS-10285 Commit: 4b527527a79f9c49e706cf06cff962a22d0332d5 Parents: 3797331 Author: Uma Maheswara Rao G Authored: Tue Jan 23 16:19:46 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Thu Jul 12 17:03:43 2018 +0530 -- .../hdfs/server/balancer/NameNodeConnector.java | 8 + .../hdfs/server/common/sps/BlockDispatcher.java | 186 + .../sps/BlockMovementAttemptFinished.java | 80 ++ .../server/common/sps/BlockMovementStatus.java | 53 .../common/sps/BlockStorageMovementTracker.java | 184 + .../sps/BlocksMovementsStatusHandler.java | 95 +++ .../hdfs/server/common/sps/package-info.java| 27 ++ .../datanode/BlockStorageMovementTracker.java | 186 - .../datanode/StoragePolicySatisfyWorker.java| 271 ++- .../hdfs/server/namenode/FSNamesystem.java | 4 +- .../namenode/sps/BlockMoveTaskHandler.java | 3 +- .../sps/BlockStorageMovementAttemptedItems.java | 12 +- .../IntraSPSNameNodeBlockMoveTaskHandler.java | 3 +- .../hdfs/server/namenode/sps/SPSService.java| 14 +- .../namenode/sps/StoragePolicySatisfier.java| 30 +- .../sps/ExternalSPSBlockMoveTaskHandler.java| 233 .../TestBlockStorageMovementAttemptedItems.java | 2 +- .../sps/TestStoragePolicySatisfier.java | 6 +- .../sps/TestExternalStoragePolicySatisfier.java | 69 - 19 files changed, 997 insertions(+), 469 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b527527/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index b0dd779..6bfbbb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -269,6 +269,14 @@ public class NameNodeConnector implements Closeable { } } + /** + * Returns fallbackToSimpleAuth. This will be true or false during calls to + * indicate if a secure client falls back to simple auth. + */ + public AtomicBoolean getFallbackToSimpleAuth() { +return fallbackToSimpleAuth; + } + @Override public void close() { keyManager.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b527527/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java new file mode 100644 index 000..f87fcae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common.sps; + +import static org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import