hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-09-14 Thread sjlee 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 8ebf8965d -> a1272d5fc


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.

(cherry picked from commit 61e1348fe479872b0eb1b53c50b248b33b37d77b)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1272d5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1272d5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1272d5f

Branch: refs/heads/branch-2.6
Commit: a1272d5fc019608d3ddca2c8ed2e03f2ddc7a0d7
Parents: 8ebf896
Author: Kihwal Lee 
Authored: Thu Aug 18 16:38:25 2016 -0500
Committer: Sangjin Lee 
Committed: Wed Sep 14 14:30:41 2016 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../server/namenode/FSImageFormatPBINode.java   | 17 ++--
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 43 
 4 files changed, 62 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272d5f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f508e84..1cc15fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -47,6 +47,9 @@ Release 2.6.5 - UNRELEASED
 
 HDFS-9696. Garbage snapshot records linger forever. (kihwal)
 
+HDFS-10763. Open files can leak permanently due to inconsistent
+lease update (kihwal)
+
 Release 2.6.4 - 2016-02-11
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272d5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 8442202..525f488 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -179,11 +179,13 @@ public final class FSImageFormatPBINode {
 private final FSDirectory dir;
 private final FSNamesystem fsn;
 private final FSImageFormatProtobuf.Loader parent;
+private final List ucFiles;
 
 Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
   this.fsn = fsn;
   this.dir = fsn.dir;
   this.parent = parent;
+  this.ucFiles = new ArrayList();
 }
 
 void loadINodeDirectorySection(InputStream in) throws IOException {
@@ -227,17 +229,25 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+ // This section is consumed, but not actually used for restoring leases.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
+  }
+
+  // Add a lease for each and every file under construction.
+  for (INodeFile file : ucFiles) {
 FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
 Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath());
+String path = file.getFullPathName();
+// Skip the deleted files in snapshot. This leaks UC inodes that are
+// deleted from the current view.
+if (path.startsWith("/")) {
+  fsn.leaseManager.addLease(uc.getClientName(), path);
+}
   }
 }
 
@@ -304,6 +314,7 @@ public final class FSImageFormatPBINode {
 
   // under-construction information
   if (f.hasFileUC()) {
+ucFiles.add(file);
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
 if (blocks.length > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272d5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-proj

[07/50] [abbrv] hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-19 Thread aengineer 
HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d

Branch: refs/heads/HDFS-7240
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee 
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:28:09 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   // replace the last block of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManage

hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-18 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 eae9b691c -> 61e1348fe


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61e1348f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61e1348f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61e1348f

Branch: refs/heads/branch-2.7
Commit: 61e1348fe479872b0eb1b53c50b248b33b37d77b
Parents: eae9b69
Author: Kihwal Lee 
Authored: Thu Aug 18 16:38:25 2016 -0500
Committer: Kihwal Lee 
Committed: Thu Aug 18 16:39:35 2016 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../server/namenode/FSImageFormatPBINode.java   | 17 ++--
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 43 
 4 files changed, 62 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/61e1348f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0ff7bba..5ca44be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -18,6 +18,9 @@ Release 2.7.4 - UNRELEASED
 HDFS-10691. FileDistribution fails in hdfs oiv command due to
 ArrayIndexOutOfBoundsException. (Yiqun Lin via aajisaka)
 
+HDFS-10763. Open files can leak permanently due to inconsistent
+lease update (kihwal)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61e1348f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index dfd150a..ed941ef 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -220,11 +220,13 @@ public final class FSImageFormatPBINode {
 private final FSDirectory dir;
 private final FSNamesystem fsn;
 private final FSImageFormatProtobuf.Loader parent;
+private final List ucFiles;
 
 Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
   this.fsn = fsn;
   this.dir = fsn.dir;
   this.parent = parent;
+  this.ucFiles = new ArrayList();
 }
 
 void loadINodeDirectorySection(InputStream in) throws IOException {
@@ -268,17 +270,25 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+ // This section is consumed, but not actually used for restoring leases.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
+  }
+
+  // Add a lease for each and every file under construction.
+  for (INodeFile file : ucFiles) {
 FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
 Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath());
+String path = file.getFullPathName();
+// Skip the deleted files in snapshot. This leaks UC inodes that are
+// deleted from the current view.
+if (path.startsWith("/")) {
+  fsn.leaseManager.addLease(uc.getClientName(), path);
+}
   }
 }
 
@@ -346,6 +356,7 @@ public final class FSImageFormatPBINode {
 
   // under-construction information
   if (f.hasFileUC()) {
+ucFiles.add(file);
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
 if (blocks.length > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61e1348f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop

[18/50] [abbrv] hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-18 Thread cnauroth 
HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d

Branch: refs/heads/HADOOP-13345
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee 
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:28:09 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   // replace the last block of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseMan

[16/50] [abbrv] hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-17 Thread subru 
HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d

Branch: refs/heads/YARN-2915
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee 
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:28:09 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   // replace the last block of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManage

[05/50] [abbrv] hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-17 Thread wangda 
HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d

Branch: refs/heads/YARN-3368
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee 
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:28:09 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   // replace the last block of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManage

hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-15 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 7519956c3 -> 71d0e4fca


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71d0e4fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71d0e4fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71d0e4fc

Branch: refs/heads/branch-2.7
Commit: 71d0e4fca79d4305fe35e44b614703d3b9883017
Parents: 7519956
Author: Kihwal Lee 
Authored: Mon Aug 15 17:40:07 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:40:07 2016 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../server/namenode/FSImageFormatPBINode.java   | 12 --
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 43 
 4 files changed, 57 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71d0e4fc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0ff7bba..a4fcf86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -255,6 +255,9 @@ Release 2.7.3 - UNRELEASED
 
 HDFS-9696. Garbage snapshot records linger forever. (kihwal)
 
+HDFS-10763. Open files can leak permanently due to inconsistent lease
+update. (kihwal)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71d0e4fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index dfd150a..7206b2f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -220,11 +220,13 @@ public final class FSImageFormatPBINode {
 private final FSDirectory dir;
 private final FSNamesystem fsn;
 private final FSImageFormatProtobuf.Loader parent;
+private final List ucFiles;
 
 Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) {
   this.fsn = fsn;
   this.dir = fsn.dir;
   this.parent = parent;
+  this.ucFiles = new ArrayList();
 }
 
 void loadINodeDirectorySection(InputStream in) throws IOException {
@@ -268,17 +270,20 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+ // This section is consumed, but not actually used for restoring leases.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
+  }
+
+  // Add a lease for each and every file under construction.
+  for (INodeFile file : ucFiles) {
 FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
 Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(), entry.getFullPath());
+fsn.leaseManager.addLease(uc.getClientName(), file.getFullPathName());
   }
 }
 
@@ -346,6 +351,7 @@ public final class FSImageFormatPBINode {
 
   // under-construction information
   if (f.hasFileUC()) {
+ucFiles.add(file);
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
 if (blocks.length > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71d0e4fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index bef296a..3887ac3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/mai

hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-15 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6fed7b9c4 -> 1a8280edd


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.

(cherry picked from commit 864f878d5912c82f3204f1582cfb7eb7c9f1a1da)
(cherry picked from commit e78db7d2a430983807750666fb72ebd5c97ce867)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a8280ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a8280ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a8280ed

Branch: refs/heads/branch-2.8
Commit: 1a8280edde5cff2393336d0a1cde4a532153f50a
Parents: 6fed7b9
Author: Kihwal Lee 
Authored: Mon Aug 15 17:37:39 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:37:39 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a8280ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 3b01f6b..80879b0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -278,18 +278,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -360,6 +356,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   lastBlk.convertToBlockUnderConstruction(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a8280ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 2948ece..6cba82e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3339,7 +3339,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3350,6 +3349,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 waitForLoadingFSImage();
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a8280ed/hadoop-hdfs-projec

hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-15 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1ef8d7a63 -> e78db7d2a


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.

(cherry picked from commit 864f878d5912c82f3204f1582cfb7eb7c9f1a1da)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e78db7d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e78db7d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e78db7d2

Branch: refs/heads/branch-2
Commit: e78db7d2a430983807750666fb72ebd5c97ce867
Parents: 1ef8d7a
Author: Kihwal Lee 
Authored: Mon Aug 15 17:33:16 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:33:16 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e78db7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 3b01f6b..80879b0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -278,18 +278,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -360,6 +356,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   lastBlk.convertToBlockUnderConstruction(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e78db7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 9b4be65..2d9a069 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3275,7 +3275,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3286,6 +3285,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e78db7d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/a

hadoop git commit: HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.

2016-08-15 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 03dea65e0 -> 864f878d5


HDFS-10763. Open files can leak permanently due to inconsistent lease update. 
Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d

Branch: refs/heads/trunk
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee 
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee 
Committed: Mon Aug 15 17:28:09 2016 -0500

--
 .../server/namenode/FSImageFormatPBINode.java   | 10 ++---
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/TestLeaseManager.java  | 47 
 3 files changed, 53 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
  * Load the under-construction files section, and update the lease map
  */
 void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+  // Leases are added when the inode section is loaded. This section is
+  // still read in for compatibility reasons.
   while (true) {
 FileUnderConstructionEntry entry = FileUnderConstructionEntry
 .parseDelimitedFrom(in);
 if (entry == null) {
   break;
 }
-// update the lease manager
-INodeFile file = dir.getInode(entry.getInodeId()).asFile();
-FileUnderConstructionFeature uc = 
file.getFileUnderConstructionFeature();
-Preconditions.checkState(uc != null); // file must be 
under-construction
-fsn.leaseManager.addLease(uc.getClientName(),
-entry.getInodeId());
   }
 }
 
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
   if (f.hasFileUC()) {
 INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
 file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+// update the lease manager
+fsn.leaseManager.addLease(uc.getClientName(), file.getId());
 if (blocks.length > 0) {
   BlockInfo lastBlk = file.getLastBlock();
   // replace the last block of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   throw new IOException("Cannot finalize file " + src
   + " because it is not under construction");
 }
-leaseManager.removeLease(uc.getClientName(), pendingFile);
 
 pendingFile.recordModification(latestSnapshot);
 
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 allowCommittedBlock? numCommittedAllowed: 0,
 blockManager.getMinReplication());
 
+leaseManager.removeLease(uc.getClientName(), pendingFile);
+
 // close file and persist block allocations for this file
 closeFile(src, pendingFile);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
 
b/hadoop-hdfs-project/had