[39/50] [abbrv] hadoop git commit: HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore.
HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d690c0b7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d690c0b7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d690c0b7 Branch: refs/heads/HDFS-10285 Commit: d690c0b79a586242d15b621faab616fa3fd0960b Parents: 7eb5942 Author: Rakesh Radhakrishnan Authored: Mon Jan 29 03:10:48 2018 +0530 Committer: Uma Maheswara Rao Gangumalla Committed: Thu Aug 9 20:47:23 2018 -0700 -- .../hadoop-hdfs/src/main/bin/hdfs | 5 + .../server/blockmanagement/BlockManager.java| 9 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 2 +- .../hdfs/server/namenode/sps/Context.java | 5 - .../namenode/sps/IntraSPSNameNodeContext.java | 4 - .../sps/IntraSPSNameNodeFileIdCollector.java| 12 +- .../hdfs/server/namenode/sps/SPSPathIds.java| 1 + .../namenode/sps/StoragePolicySatisfier.java| 83 +++- .../sps/ExternalSPSBlockMoveTaskHandler.java| 2 +- .../hdfs/server/sps/ExternalSPSContext.java | 57 +--- .../server/sps/ExternalSPSFileIDCollector.java | 12 +- .../sps/ExternalStoragePolicySatisfier.java | 130 +++ .../src/site/markdown/ArchivalStorage.md| 10 +- .../sps/TestStoragePolicySatisfier.java | 22 ++-- .../sps/TestExternalStoragePolicySatisfier.java | 33 +++-- 15 files changed, 259 insertions(+), 128 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d690c0b7/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index bc6e7a4..94426a5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -63,6 +63,7 @@ function hadoop_usage hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode" hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot" hadoop_add_subcommand "storagepolicies" admin "list/get/set/satisfyStoragePolicy block storage policies" + hadoop_add_subcommand "sps" daemon "run external storagepolicysatisfier" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false @@ -201,6 +202,10 @@ function hdfscmd_case storagepolicies) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin ;; +sps) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.sps.ExternalStoragePolicySatisfier +;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d690c0b7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index ac6d44b..4ea64a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -94,6 +94,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeBlockMoveTaskHandler; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeFileIdCollector; import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds; import org.apache.hadoop.hdfs.server.namenode.sps.SPSService; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; @@ -5106,9 +5109,15 @@ public class BlockManager implements BlockStatsMXBean { return; } updateSPSMode(StoragePolicySatisfierMode.INTERNAL); +sps.init(new IntraSPSNameNodeContext(this.namesystem, this, sps), +new IntraSPSNameNodeFileIdCollector(this.namesystem.getFSDirectory(), +sps), +
[39/50] [abbrv] hadoop git commit: HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore.
HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5c0b053 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5c0b053 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5c0b053 Branch: refs/heads/HDFS-10285 Commit: e5c0b053ad15fbc8ba4b909daee5973e621efcf7 Parents: 5255f96 Author: Rakesh Radhakrishnan Authored: Mon Jan 29 03:10:48 2018 +0530 Committer: Rakesh Radhakrishnan Committed: Thu Jul 19 22:49:28 2018 +0530 -- .../hadoop-hdfs/src/main/bin/hdfs | 5 + .../server/blockmanagement/BlockManager.java| 9 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 2 +- .../hdfs/server/namenode/sps/Context.java | 5 - .../namenode/sps/IntraSPSNameNodeContext.java | 4 - .../sps/IntraSPSNameNodeFileIdCollector.java| 12 +- .../hdfs/server/namenode/sps/SPSPathIds.java| 1 + .../namenode/sps/StoragePolicySatisfier.java| 83 +++- .../sps/ExternalSPSBlockMoveTaskHandler.java| 2 +- .../hdfs/server/sps/ExternalSPSContext.java | 57 +--- .../server/sps/ExternalSPSFileIDCollector.java | 12 +- .../sps/ExternalStoragePolicySatisfier.java | 130 +++ .../src/site/markdown/ArchivalStorage.md| 10 +- .../sps/TestStoragePolicySatisfier.java | 22 ++-- .../sps/TestExternalStoragePolicySatisfier.java | 33 +++-- 15 files changed, 259 insertions(+), 128 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5c0b053/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index bc6e7a4..94426a5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -63,6 +63,7 @@ function hadoop_usage hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode" hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot" hadoop_add_subcommand "storagepolicies" admin "list/get/set/satisfyStoragePolicy block storage policies" + hadoop_add_subcommand "sps" daemon "run external storagepolicysatisfier" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false @@ -201,6 +202,10 @@ function hdfscmd_case storagepolicies) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin ;; +sps) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.sps.ExternalStoragePolicySatisfier +;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5c0b053/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index daaa7a3..f348a33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -94,6 +94,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeBlockMoveTaskHandler; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeFileIdCollector; import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds; import org.apache.hadoop.hdfs.server.namenode.sps.SPSService; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; @@ -5098,9 +5101,15 @@ public class BlockManager implements BlockStatsMXBean { return; } updateSPSMode(StoragePolicySatisfierMode.INTERNAL); +sps.init(new IntraSPSNameNodeContext(this.namesystem, this, sps), +new IntraSPSNameNodeFileIdCollector(this.namesystem.getFSDirectory(), +sps), +new
[39/50] [abbrv] hadoop git commit: HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore.
HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f3d77ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f3d77ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f3d77ae Branch: refs/heads/HDFS-10285 Commit: 5f3d77aec75ef503fe45674afb8fd1086cd42b0d Parents: a6ba640 Author: Rakesh Radhakrishnan Authored: Mon Jan 29 03:10:48 2018 +0530 Committer: Rakesh Radhakrishnan Committed: Sun Jul 15 20:22:42 2018 +0530 -- .../hadoop-hdfs/src/main/bin/hdfs | 5 + .../server/blockmanagement/BlockManager.java| 9 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 2 +- .../hdfs/server/namenode/sps/Context.java | 5 - .../namenode/sps/IntraSPSNameNodeContext.java | 4 - .../sps/IntraSPSNameNodeFileIdCollector.java| 12 +- .../hdfs/server/namenode/sps/SPSPathIds.java| 1 + .../namenode/sps/StoragePolicySatisfier.java| 83 +++- .../sps/ExternalSPSBlockMoveTaskHandler.java| 2 +- .../hdfs/server/sps/ExternalSPSContext.java | 57 +--- .../server/sps/ExternalSPSFileIDCollector.java | 12 +- .../sps/ExternalStoragePolicySatisfier.java | 130 +++ .../src/site/markdown/ArchivalStorage.md| 10 +- .../sps/TestStoragePolicySatisfier.java | 22 ++-- .../sps/TestExternalStoragePolicySatisfier.java | 33 +++-- 15 files changed, 259 insertions(+), 128 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f3d77ae/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index bc6e7a4..94426a5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -63,6 +63,7 @@ function hadoop_usage hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode" hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot" hadoop_add_subcommand "storagepolicies" admin "list/get/set/satisfyStoragePolicy block storage policies" + hadoop_add_subcommand "sps" daemon "run external storagepolicysatisfier" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false @@ -201,6 +202,10 @@ function hdfscmd_case storagepolicies) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin ;; +sps) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.sps.ExternalStoragePolicySatisfier +;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f3d77ae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index daaa7a3..f348a33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -94,6 +94,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeBlockMoveTaskHandler; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeFileIdCollector; import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds; import org.apache.hadoop.hdfs.server.namenode.sps.SPSService; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; @@ -5098,9 +5101,15 @@ public class BlockManager implements BlockStatsMXBean { return; } updateSPSMode(StoragePolicySatisfierMode.INTERNAL); +sps.init(new IntraSPSNameNodeContext(this.namesystem, this, sps), +new IntraSPSNameNodeFileIdCollector(this.namesystem.getFSDirectory(), +sps), +new
[39/50] [abbrv] hadoop git commit: HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore.
HDFS-13050: [SPS]: Create start/stop script to start external SPS process. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5213c199 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5213c199 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5213c199 Branch: refs/heads/HDFS-10285 Commit: 5213c199544b51522a7ca6e9812fa2c6d3b10ce2 Parents: 659bfd7 Author: Rakesh Radhakrishnan Authored: Mon Jan 29 03:10:48 2018 +0530 Committer: Rakesh Radhakrishnan Committed: Thu Jul 12 17:04:07 2018 +0530 -- .../hadoop-hdfs/src/main/bin/hdfs | 5 + .../server/blockmanagement/BlockManager.java| 9 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 2 +- .../hdfs/server/namenode/sps/Context.java | 5 - .../namenode/sps/IntraSPSNameNodeContext.java | 4 - .../sps/IntraSPSNameNodeFileIdCollector.java| 12 +- .../hdfs/server/namenode/sps/SPSPathIds.java| 1 + .../namenode/sps/StoragePolicySatisfier.java| 83 +++- .../sps/ExternalSPSBlockMoveTaskHandler.java| 2 +- .../hdfs/server/sps/ExternalSPSContext.java | 57 +--- .../server/sps/ExternalSPSFileIDCollector.java | 12 +- .../sps/ExternalStoragePolicySatisfier.java | 130 +++ .../src/site/markdown/ArchivalStorage.md| 10 +- .../sps/TestStoragePolicySatisfier.java | 22 ++-- .../sps/TestExternalStoragePolicySatisfier.java | 33 +++-- 15 files changed, 259 insertions(+), 128 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5213c199/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index bc6e7a4..94426a5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -63,6 +63,7 @@ function hadoop_usage hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode" hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot" hadoop_add_subcommand "storagepolicies" admin "list/get/set/satisfyStoragePolicy block storage policies" + hadoop_add_subcommand "sps" daemon "run external storagepolicysatisfier" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false @@ -201,6 +202,10 @@ function hdfscmd_case storagepolicies) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin ;; +sps) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.sps.ExternalStoragePolicySatisfier +;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5213c199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index daaa7a3..f348a33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -94,6 +94,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeBlockMoveTaskHandler; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext; +import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeFileIdCollector; import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds; import org.apache.hadoop.hdfs.server.namenode.sps.SPSService; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; @@ -5098,9 +5101,15 @@ public class BlockManager implements BlockStatsMXBean { return; } updateSPSMode(StoragePolicySatisfierMode.INTERNAL); +sps.init(new IntraSPSNameNodeContext(this.namesystem, this, sps), +new IntraSPSNameNodeFileIdCollector(this.namesystem.getFSDirectory(), +sps), +new