[02/50] [abbrv] hadoop git commit: HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy for a file. Contributed by Uma Maheswara Rao G

2018-07-19 Thread rakeshr 
HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy 
for a file. Contributed by Uma Maheswara Rao G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f3da78d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f3da78d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f3da78d

Branch: refs/heads/HDFS-10285
Commit: 8f3da78d6cda296207fb5bd3613f0387e5fb8e9d
Parents: 6081ddc
Author: Rakesh Radhakrishnan 
Authored: Thu Apr 20 23:14:36 2017 +0530
Committer: Rakesh Radhakrishnan 
Committed: Thu Jul 19 22:45:37 2018 +0530

--
 .../BlockStorageMovementAttemptedItems.java |   2 +-
 .../server/namenode/StoragePolicySatisfier.java | 116 ++-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  35 ++
 .../TestPersistentStoragePolicySatisfier.java   |  52 +
 .../namenode/TestStoragePolicySatisfier.java|  76 
 5 files changed, 225 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3da78d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index f2406da..bf7859c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -333,7 +333,7 @@ public class BlockStorageMovementAttemptedItems {
   + "doesn't exists in storageMovementAttemptedItems list",
   storageMovementAttemptedResult.getTrackId());
   // Remove xattr for the track id.
-  this.sps.notifyBlkStorageMovementFinished(
+  this.sps.postBlkStorageMovementCleanup(
   storageMovementAttemptedResult.getTrackId());
 }
 break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3da78d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 8be0a2a..3b20314 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -79,6 +79,27 @@ public class StoragePolicySatisfier implements Runnable {
   private final BlockStorageMovementAttemptedItems storageMovementsMonitor;
   private volatile boolean isRunning = false;
 
+  /**
+   * Represents the collective analysis status for all blocks.
+   */
+  private enum BlocksMovingAnalysisStatus {
+// Represents that, the analysis skipped due to some conditions. A such
+// condition is if block collection is in incomplete state.
+ANALYSIS_SKIPPED_FOR_RETRY,
+// Represents that, all block storage movement needed blocks found its
+// targets.
+ALL_BLOCKS_TARGETS_PAIRED,
+// Represents that, only fewer or none of the block storage movement needed
+// block found its eligible targets.
+FEW_BLOCKS_TARGETS_PAIRED,
+// Represents that, none of the blocks found for block storage movements.
+BLOCKS_ALREADY_SATISFIED,
+// Represents that, the analysis skipped due to some conditions.
+// Example conditions are if no blocks really exists in block collection or
+// if analysis is not required on ec files with unsuitable storage policies
+BLOCKS_TARGET_PAIRING_SKIPPED;
+  }
+
   public StoragePolicySatisfier(final Namesystem namesystem,
   final BlockStorageMovementNeeded storageMovementNeeded,
   final BlockManager blkManager, Configuration conf) {
@@ -208,10 +229,31 @@ public class StoragePolicySatisfier implements Runnable {
 namesystem.getBlockCollection(blockCollectionID);
 // Check blockCollectionId existence.
 if (blockCollection != null) {
-  boolean allBlockLocsAttemptedToSatisfy =
-  
computeAndAssignStorageMismatchedBlocksToDNs(blockCollection);
-  this.storageMovementsMonitor
-

[02/50] [abbrv] hadoop git commit: HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy for a file. Contributed by Uma Maheswara Rao G

2018-07-16 Thread rakeshr 
HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy 
for a file. Contributed by Uma Maheswara Rao G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1916bb80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1916bb80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1916bb80

Branch: refs/heads/HDFS-10285
Commit: 1916bb805e74935e76f8cd3a9938c3503940519d
Parents: 2363781
Author: Rakesh Radhakrishnan 
Authored: Thu Apr 20 23:14:36 2017 +0530
Committer: Rakesh Radhakrishnan 
Committed: Sun Jul 15 20:17:57 2018 +0530

--
 .../BlockStorageMovementAttemptedItems.java |   2 +-
 .../server/namenode/StoragePolicySatisfier.java | 116 ++-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  35 ++
 .../TestPersistentStoragePolicySatisfier.java   |  52 +
 .../namenode/TestStoragePolicySatisfier.java|  76 
 5 files changed, 225 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1916bb80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index f2406da..bf7859c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -333,7 +333,7 @@ public class BlockStorageMovementAttemptedItems {
   + "doesn't exists in storageMovementAttemptedItems list",
   storageMovementAttemptedResult.getTrackId());
   // Remove xattr for the track id.
-  this.sps.notifyBlkStorageMovementFinished(
+  this.sps.postBlkStorageMovementCleanup(
   storageMovementAttemptedResult.getTrackId());
 }
 break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1916bb80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 8be0a2a..3b20314 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -79,6 +79,27 @@ public class StoragePolicySatisfier implements Runnable {
   private final BlockStorageMovementAttemptedItems storageMovementsMonitor;
   private volatile boolean isRunning = false;
 
+  /**
+   * Represents the collective analysis status for all blocks.
+   */
+  private enum BlocksMovingAnalysisStatus {
+// Represents that, the analysis skipped due to some conditions. A such
+// condition is if block collection is in incomplete state.
+ANALYSIS_SKIPPED_FOR_RETRY,
+// Represents that, all block storage movement needed blocks found its
+// targets.
+ALL_BLOCKS_TARGETS_PAIRED,
+// Represents that, only fewer or none of the block storage movement needed
+// block found its eligible targets.
+FEW_BLOCKS_TARGETS_PAIRED,
+// Represents that, none of the blocks found for block storage movements.
+BLOCKS_ALREADY_SATISFIED,
+// Represents that, the analysis skipped due to some conditions.
+// Example conditions are if no blocks really exists in block collection or
+// if analysis is not required on ec files with unsuitable storage policies
+BLOCKS_TARGET_PAIRING_SKIPPED;
+  }
+
   public StoragePolicySatisfier(final Namesystem namesystem,
   final BlockStorageMovementNeeded storageMovementNeeded,
   final BlockManager blkManager, Configuration conf) {
@@ -208,10 +229,31 @@ public class StoragePolicySatisfier implements Runnable {
 namesystem.getBlockCollection(blockCollectionID);
 // Check blockCollectionId existence.
 if (blockCollection != null) {
-  boolean allBlockLocsAttemptedToSatisfy =
-  
computeAndAssignStorageMismatchedBlocksToDNs(blockCollection);
-  this.storageMovementsMonitor
-

[02/50] [abbrv] hadoop git commit: HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy for a file. Contributed by Uma Maheswara Rao G

2018-07-12 Thread rakeshr 
HDFS-11572. [SPS]: SPS should clean Xattrs when no blocks required to satisfy 
for a file. Contributed by Uma Maheswara Rao G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95f64fb6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95f64fb6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95f64fb6

Branch: refs/heads/HDFS-10285
Commit: 95f64fb644f15adb29906f16e93c55a23a58ed57
Parents: d37ae23
Author: Rakesh Radhakrishnan 
Authored: Thu Apr 20 23:14:36 2017 +0530
Committer: Rakesh Radhakrishnan 
Committed: Thu Jul 12 17:00:25 2018 +0530

--
 .../BlockStorageMovementAttemptedItems.java |   2 +-
 .../server/namenode/StoragePolicySatisfier.java | 116 ++-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  35 ++
 .../TestPersistentStoragePolicySatisfier.java   |  52 +
 .../namenode/TestStoragePolicySatisfier.java|  76 
 5 files changed, 225 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/95f64fb6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index f2406da..bf7859c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -333,7 +333,7 @@ public class BlockStorageMovementAttemptedItems {
   + "doesn't exists in storageMovementAttemptedItems list",
   storageMovementAttemptedResult.getTrackId());
   // Remove xattr for the track id.
-  this.sps.notifyBlkStorageMovementFinished(
+  this.sps.postBlkStorageMovementCleanup(
   storageMovementAttemptedResult.getTrackId());
 }
 break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/95f64fb6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 8be0a2a..3b20314 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -79,6 +79,27 @@ public class StoragePolicySatisfier implements Runnable {
   private final BlockStorageMovementAttemptedItems storageMovementsMonitor;
   private volatile boolean isRunning = false;
 
+  /**
+   * Represents the collective analysis status for all blocks.
+   */
+  private enum BlocksMovingAnalysisStatus {
+// Represents that, the analysis skipped due to some conditions. A such
+// condition is if block collection is in incomplete state.
+ANALYSIS_SKIPPED_FOR_RETRY,
+// Represents that, all block storage movement needed blocks found its
+// targets.
+ALL_BLOCKS_TARGETS_PAIRED,
+// Represents that, only fewer or none of the block storage movement needed
+// block found its eligible targets.
+FEW_BLOCKS_TARGETS_PAIRED,
+// Represents that, none of the blocks found for block storage movements.
+BLOCKS_ALREADY_SATISFIED,
+// Represents that, the analysis skipped due to some conditions.
+// Example conditions are if no blocks really exists in block collection or
+// if analysis is not required on ec files with unsuitable storage policies
+BLOCKS_TARGET_PAIRING_SKIPPED;
+  }
+
   public StoragePolicySatisfier(final Namesystem namesystem,
   final BlockStorageMovementNeeded storageMovementNeeded,
   final BlockManager blkManager, Configuration conf) {
@@ -208,10 +229,31 @@ public class StoragePolicySatisfier implements Runnable {
 namesystem.getBlockCollection(blockCollectionID);
 // Check blockCollectionId existence.
 if (blockCollection != null) {
-  boolean allBlockLocsAttemptedToSatisfy =
-  
computeAndAssignStorageMismatchedBlocksToDNs(blockCollection);
-  this.storageMovementsMonitor
-