[37/50] [abbrv] hadoop git commit: HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R.
HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a2bbde9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a2bbde9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a2bbde9 Branch: refs/heads/HDFS-10285 Commit: 8a2bbde97125b0aa73a88b1d8f79302966efccaf Parents: 941180d Author: Uma Maheswara Rao G Authored: Fri Jan 26 08:57:29 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Thu Jul 19 22:49:04 2018 +0530 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 39 .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +- .../server/blockmanagement/BlockManager.java| 105 +++--- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../hdfs/server/namenode/FSTreeTraverser.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 34 ++-- .../sps/BlockStorageMovementNeeded.java | 2 +- .../namenode/sps/IntraSPSNameNodeContext.java | 3 + .../hdfs/server/namenode/sps/SPSService.java| 4 +- .../namenode/sps/StoragePolicySatisfier.java| 17 +- .../server/sps/ExternalSPSFileIDCollector.java | 32 ++- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 16 +- .../src/main/resources/hdfs-default.xml | 11 +- .../src/site/markdown/ArchivalStorage.md| 17 +- .../TestStoragePolicySatisfyWorker.java | 5 +- .../hadoop/hdfs/server/mover/TestMover.java | 45 +++-- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../namenode/TestNameNodeReconfigure.java | 105 +- .../TestPersistentStoragePolicySatisfier.java | 9 +- .../TestStoragePolicySatisfierWithHA.java | 12 +- .../sps/TestStoragePolicySatisfier.java | 202 +++ ...stStoragePolicySatisfierWithStripedFile.java | 17 +- .../sps/TestExternalStoragePolicySatisfier.java | 112 +++--- .../hdfs/tools/TestStoragePolicyCommands.java | 5 +- .../TestStoragePolicySatisfyAdminCommands.java | 14 +- 25 files changed, 500 insertions(+), 327 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a2bbde9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index aabcdd9..ab48dcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -129,6 +129,45 @@ public final class HdfsConstants { } /** + * Storage policy satisfier service modes. + */ + public enum StoragePolicySatisfierMode { + +/** + * This mode represents that SPS service is running inside Namenode and can + * accept any SPS call request. + */ +INTERNAL, + +/** + * This mode represents that SPS service is running outside Namenode as an + * external service and can accept any SPS call request. + */ +EXTERNAL, + +/** + * This mode represents that SPS service is disabled and cannot accept any + * SPS call request. + */ +NONE; + +private static final Map MAP = +new HashMap<>(); + +static { + for (StoragePolicySatisfierMode a : values()) { +MAP.put(a.name(), a); + } +} + +/** Convert the given String to a StoragePolicySatisfierMode. */ +public static StoragePolicySatisfierMode fromString(String s) { + return MAP.get(StringUtils.toUpperCase(s)); +} + } + + + /** * Storage policy satisfy path status. */ public enum StoragePolicySatisfyPathStatus { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a2bbde9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fc95d8c..0d93ff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import
[37/50] [abbrv] hadoop git commit: HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R.
HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b49a0c0d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b49a0c0d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b49a0c0d Branch: refs/heads/HDFS-10285 Commit: b49a0c0d28a66c1fbb29a39c06555d03856cc1b4 Parents: 0c97485 Author: Uma Maheswara Rao G Authored: Fri Jan 26 08:57:29 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Sun Jul 15 20:22:19 2018 +0530 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 39 .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +- .../server/blockmanagement/BlockManager.java| 105 +++--- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../hdfs/server/namenode/FSTreeTraverser.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 34 ++-- .../sps/BlockStorageMovementNeeded.java | 2 +- .../namenode/sps/IntraSPSNameNodeContext.java | 3 + .../hdfs/server/namenode/sps/SPSService.java| 4 +- .../namenode/sps/StoragePolicySatisfier.java| 17 +- .../server/sps/ExternalSPSFileIDCollector.java | 32 ++- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 16 +- .../src/main/resources/hdfs-default.xml | 11 +- .../src/site/markdown/ArchivalStorage.md| 17 +- .../TestStoragePolicySatisfyWorker.java | 5 +- .../hadoop/hdfs/server/mover/TestMover.java | 45 +++-- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../namenode/TestNameNodeReconfigure.java | 105 +- .../TestPersistentStoragePolicySatisfier.java | 9 +- .../TestStoragePolicySatisfierWithHA.java | 12 +- .../sps/TestStoragePolicySatisfier.java | 202 +++ ...stStoragePolicySatisfierWithStripedFile.java | 17 +- .../sps/TestExternalStoragePolicySatisfier.java | 112 +++--- .../hdfs/tools/TestStoragePolicyCommands.java | 5 +- .../TestStoragePolicySatisfyAdminCommands.java | 14 +- 25 files changed, 500 insertions(+), 327 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b49a0c0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index aabcdd9..ab48dcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -129,6 +129,45 @@ public final class HdfsConstants { } /** + * Storage policy satisfier service modes. + */ + public enum StoragePolicySatisfierMode { + +/** + * This mode represents that SPS service is running inside Namenode and can + * accept any SPS call request. + */ +INTERNAL, + +/** + * This mode represents that SPS service is running outside Namenode as an + * external service and can accept any SPS call request. + */ +EXTERNAL, + +/** + * This mode represents that SPS service is disabled and cannot accept any + * SPS call request. + */ +NONE; + +private static final Map MAP = +new HashMap<>(); + +static { + for (StoragePolicySatisfierMode a : values()) { +MAP.put(a.name(), a); + } +} + +/** Convert the given String to a StoragePolicySatisfierMode. */ +public static StoragePolicySatisfierMode fromString(String s) { + return MAP.get(StringUtils.toUpperCase(s)); +} + } + + + /** * Storage policy satisfy path status. */ public enum StoragePolicySatisfyPathStatus { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b49a0c0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fc95d8c..0d93ff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import
[37/50] [abbrv] hadoop git commit: HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R.
HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/547537ec Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/547537ec Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/547537ec Branch: refs/heads/HDFS-10285 Commit: 547537ec7423b9a4f76067482780273b353eb779 Parents: 4b52752 Author: Uma Maheswara Rao G Authored: Fri Jan 26 08:57:29 2018 -0800 Committer: Rakesh Radhakrishnan Committed: Thu Jul 12 17:03:51 2018 +0530 -- .../hadoop/hdfs/protocol/HdfsConstants.java | 39 .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +- .../server/blockmanagement/BlockManager.java| 105 +++--- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../hdfs/server/namenode/FSTreeTraverser.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 34 ++-- .../sps/BlockStorageMovementNeeded.java | 2 +- .../namenode/sps/IntraSPSNameNodeContext.java | 3 + .../hdfs/server/namenode/sps/SPSService.java| 4 +- .../namenode/sps/StoragePolicySatisfier.java| 17 +- .../server/sps/ExternalSPSFileIDCollector.java | 32 ++- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 16 +- .../src/main/resources/hdfs-default.xml | 11 +- .../src/site/markdown/ArchivalStorage.md| 17 +- .../TestStoragePolicySatisfyWorker.java | 5 +- .../hadoop/hdfs/server/mover/TestMover.java | 45 +++-- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../namenode/TestNameNodeReconfigure.java | 105 +- .../TestPersistentStoragePolicySatisfier.java | 9 +- .../TestStoragePolicySatisfierWithHA.java | 12 +- .../sps/TestStoragePolicySatisfier.java | 202 +++ ...stStoragePolicySatisfierWithStripedFile.java | 17 +- .../sps/TestExternalStoragePolicySatisfier.java | 112 +++--- .../hdfs/tools/TestStoragePolicyCommands.java | 5 +- .../TestStoragePolicySatisfyAdminCommands.java | 14 +- 25 files changed, 500 insertions(+), 327 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/547537ec/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index aabcdd9..ab48dcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -129,6 +129,45 @@ public final class HdfsConstants { } /** + * Storage policy satisfier service modes. + */ + public enum StoragePolicySatisfierMode { + +/** + * This mode represents that SPS service is running inside Namenode and can + * accept any SPS call request. + */ +INTERNAL, + +/** + * This mode represents that SPS service is running outside Namenode as an + * external service and can accept any SPS call request. + */ +EXTERNAL, + +/** + * This mode represents that SPS service is disabled and cannot accept any + * SPS call request. + */ +NONE; + +private static final Map MAP = +new HashMap<>(); + +static { + for (StoragePolicySatisfierMode a : values()) { +MAP.put(a.name(), a); + } +} + +/** Convert the given String to a StoragePolicySatisfierMode. */ +public static StoragePolicySatisfierMode fromString(String s) { + return MAP.get(StringUtils.toUpperCase(s)); +} + } + + + /** * Storage policy satisfy path status. */ public enum StoragePolicySatisfyPathStatus { http://git-wip-us.apache.org/repos/asf/hadoop/blob/547537ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fc95d8c..0d93ff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import
[37/50] [abbrv] hadoop git commit: HDFS-13057: [SPS]: Revisit configurations to make SPS service modes internal/external/none. Contributed by Rakesh R.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fb1858/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java index 0e3a5a3..2257608 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -103,8 +104,8 @@ public class TestStoragePolicySatisfierWithStripedFile { } final Configuration conf = new HdfsConfiguration(); -conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY, -true); +conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, +StoragePolicySatisfierMode.INTERNAL.toString()); initConfWithStripe(conf, defaultStripeBlockSize); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) @@ -216,8 +217,8 @@ public class TestStoragePolicySatisfierWithStripedFile { } final Configuration conf = new HdfsConfiguration(); -conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY, -true); +conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, +StoragePolicySatisfierMode.INTERNAL.toString()); initConfWithStripe(conf, defaultStripeBlockSize); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) @@ -328,8 +329,8 @@ public class TestStoragePolicySatisfierWithStripedFile { conf.set(DFSConfigKeys .DFS_STORAGE_POLICY_SATISFIER_RECHECK_TIMEOUT_MILLIS_KEY, "3000"); -conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY, -true); +conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, +StoragePolicySatisfierMode.INTERNAL.toString()); initConfWithStripe(conf, defaultStripeBlockSize); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) @@ -420,8 +421,8 @@ public class TestStoragePolicySatisfierWithStripedFile { } final Configuration conf = new HdfsConfiguration(); -conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY, -true); +conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, +StoragePolicySatisfierMode.INTERNAL.toString()); initConfWithStripe(conf, defaultStripeBlockSize); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) http://git-wip-us.apache.org/repos/asf/hadoop/blob/96fb1858/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java index 9a401bd..42b04da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode; import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.namenode.sps.BlockMovementListener; @@ -54,12 +55,19 @@ public class TestExternalStoragePolicySatisfier new StorageType[][]{{StorageType.DISK, StorageType.DISK}, {StorageType.DISK, StorageType.DISK}, {StorageType.DISK, StorageType.DISK}};