hadoop git commit: HDFS-7707. Edit log corruption due to delayed block removal again. Contributed by Yongjun Zhang (cherry picked from commit 843806d03ab1a24f191782f42eb817505228eb9f)

2015-08-31 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.1 e917ef3cf -> a1c3e6a05


HDFS-7707. Edit log corruption due to delayed block removal again. Contributed 
by Yongjun Zhang
(cherry picked from commit 843806d03ab1a24f191782f42eb817505228eb9f)

(cherry picked from commit c428d303f67bef3a7df12153947c6b0199a0938b)
(cherry picked from commit 21d8b22feef2fc9f6793d9c157391b142368e588)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1c3e6a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1c3e6a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1c3e6a0

Branch: refs/heads/branch-2.6.1
Commit: a1c3e6a058e4041ce7d1601867154b9ddd0d9c30
Parents: e917ef3
Author: Kihwal Lee 
Authored: Tue Feb 3 14:45:15 2015 -0600
Committer: Vinod Kumar Vavilapalli 
Committed: Mon Aug 31 15:55:07 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hdfs/server/namenode/FSNamesystem.java  | 14 +++--
 .../TestCommitBlockSynchronization.java |  4 ++-
 .../hdfs/server/namenode/TestDeleteRace.java| 32 +++-
 4 files changed, 43 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ec049b0..648f1ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -67,6 +67,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7575. Upgrade should generate a unique storage ID for each
 volume. (Arpit Agarwal)
 
+HDFS-7707. Edit log corruption due to delayed block removal again.
+(Yongjun Zhang via kihwal)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7077b68..8e5a2db 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -238,6 +238,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
 import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import 
org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -6864,13 +6865,22 @@ public class FSNamesystem implements Namesystem, 
FSClusterStats,
 INode tmpChild = file;
 INodeDirectory tmpParent = file.getParent();
 while (true) {
-  if (tmpParent == null ||
-  tmpParent.searchChildren(tmpChild.getLocalNameBytes()) < 0) {
+  if (tmpParent == null) {
 return true;
   }
+
+  INode childINode = tmpParent.getChild(tmpChild.getLocalNameBytes(),
+  Snapshot.CURRENT_STATE_ID);
+  if (childINode == null || !childINode.equals(tmpChild)) {
+// a newly created INode with the same name as an already deleted one
+// would be a different INode than the deleted one
+return true;
+  }
+
   if (tmpParent.isRoot()) {
 break;
   }
+
   tmpChild = tmpParent;
   tmpParent = tmpParent.getParent();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1c3e6a0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index bd71870..816c7c8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++

hadoop git commit: HDFS-7707. Edit log corruption due to delayed block removal again. Contributed by Yongjun Zhang

2015-02-03 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 21d80b3dd - 843806d03


HDFS-7707. Edit log corruption due to delayed block removal again. Contributed 
by Yongjun Zhang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/843806d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/843806d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/843806d0

Branch: refs/heads/trunk
Commit: 843806d03ab1a24f191782f42eb817505228eb9f
Parents: 21d80b3
Author: Kihwal Lee kih...@apache.org
Authored: Tue Feb 3 14:45:15 2015 -0600
Committer: Kihwal Lee kih...@apache.org
Committed: Tue Feb 3 14:45:15 2015 -0600

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hdfs/server/namenode/FSNamesystem.java  | 14 +++--
 .../TestCommitBlockSynchronization.java |  4 ++-
 .../hdfs/server/namenode/TestDeleteRace.java| 32 +++-
 4 files changed, 43 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/843806d0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d6283b9..7a0fdb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -830,6 +830,9 @@ Release 2.7.0 - UNRELEASED
 HDFS-6651. Deletion failure can leak inodes permanently.
 (Jing Zhao via wheat9)
 
+HDFS-7707. Edit log corruption due to delayed block removal again.
+(Yongjun Zhang via kihwal)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/843806d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index ebdec1b..e5ecb53 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -227,6 +227,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
 import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import 
org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -6012,13 +6013,22 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 INode tmpChild = file;
 INodeDirectory tmpParent = file.getParent();
 while (true) {
-  if (tmpParent == null ||
-  tmpParent.searchChildren(tmpChild.getLocalNameBytes())  0) {
+  if (tmpParent == null) {
 return true;
   }
+
+  INode childINode = tmpParent.getChild(tmpChild.getLocalNameBytes(),
+  Snapshot.CURRENT_STATE_ID);
+  if (childINode == null || !childINode.equals(tmpChild)) {
+// a newly created INode with the same name as an already deleted one
+// would be a different INode than the deleted one
+return true;
+  }
+
   if (tmpParent.isRoot()) {
 break;
   }
+
   tmpChild = tmpParent;
   tmpParent = tmpParent.getParent();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/843806d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index eae65cc..d88227a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -54,7 +54,9 @@ public class TestCommitBlockSynchronization {
 // set file's parent as root and put the file to inodeMap, so
 // FSNamesystem's isFileDeleted() method will return false on

hadoop git commit: HDFS-7707. Edit log corruption due to delayed block removal again. Contributed by Yongjun Zhang (cherry picked from commit 843806d03ab1a24f191782f42eb817505228eb9f)

2015-02-03 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c0b1311a9 - c428d303f


HDFS-7707. Edit log corruption due to delayed block removal again. Contributed 
by Yongjun Zhang
(cherry picked from commit 843806d03ab1a24f191782f42eb817505228eb9f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c428d303
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c428d303
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c428d303

Branch: refs/heads/branch-2
Commit: c428d303f67bef3a7df12153947c6b0199a0938b
Parents: c0b1311
Author: Kihwal Lee kih...@apache.org
Authored: Tue Feb 3 14:45:15 2015 -0600
Committer: Kihwal Lee kih...@apache.org
Committed: Tue Feb 3 14:46:24 2015 -0600

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hdfs/server/namenode/FSNamesystem.java  | 14 +++--
 .../TestCommitBlockSynchronization.java |  4 ++-
 .../hdfs/server/namenode/TestDeleteRace.java| 32 +++-
 4 files changed, 43 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c428d303/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 440be44..f9770c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.7.0 - UNRELEASED
 HDFS-6651. Deletion failure can leak inodes permanently.
 (Jing Zhao via wheat9)
 
+HDFS-7707. Edit log corruption due to delayed block removal again.
+(Yongjun Zhang via kihwal)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c428d303/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3f5cc4a..edca0c8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -231,6 +231,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
 import org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import 
org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
@@ -5999,13 +6000,22 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 INode tmpChild = file;
 INodeDirectory tmpParent = file.getParent();
 while (true) {
-  if (tmpParent == null ||
-  tmpParent.searchChildren(tmpChild.getLocalNameBytes())  0) {
+  if (tmpParent == null) {
 return true;
   }
+
+  INode childINode = tmpParent.getChild(tmpChild.getLocalNameBytes(),
+  Snapshot.CURRENT_STATE_ID);
+  if (childINode == null || !childINode.equals(tmpChild)) {
+// a newly created INode with the same name as an already deleted one
+// would be a different INode than the deleted one
+return true;
+  }
+
   if (tmpParent.isRoot()) {
 break;
   }
+
   tmpChild = tmpParent;
   tmpParent = tmpParent.getParent();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c428d303/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index eae65cc..d88227a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -54,7 +54,9 @@ public class TestCommitBlockSynchronization {
 // set file's parent as root and put the file to