hadoop git commit: HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by Vinayakumar B.

2017-08-09 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8.2 68380262f -> 5758dad28


HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by 
Vinayakumar B.

(cherry picked from commit 74bcc8d1f213ffd18056b7d0590d0cd346abaff9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5758dad2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5758dad2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5758dad2

Branch: refs/heads/branch-2.8.2
Commit: 5758dad28f70d61e5b5a58e9dda0efa47135fc81
Parents: 6838026
Author: Kihwal Lee 
Authored: Wed Aug 9 09:38:42 2017 -0500
Committer: Kihwal Lee 
Committed: Wed Aug 9 09:38:42 2017 -0500

--
 .../datanode/fsdataset/impl/FsDatasetImpl.java  | 40 +++-
 1 file changed, 22 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5758dad2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 6468d65..ce0f7c0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1010,8 +1010,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   targetVolume, blockFiles[0].getParentFile(), 0);
   newReplicaInfo.setNumBytes(blockFiles[1].length());
   // Finalize the copied files
-  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo,
-  false);
+  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
   try(AutoCloseableLock lock = datasetLock.acquire()) {
 // Increment numBlocks here as this block moved without knowing to BPS
 FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume();
@@ -1372,7 +1371,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   bumpReplicaGS(replicaInfo, newGS);
   // finalize the replica if RBW
   if (replicaInfo.getState() == ReplicaState.RBW) {
-finalizeReplica(b.getBlockPoolId(), replicaInfo, false);
+finalizeReplica(b.getBlockPoolId(), replicaInfo);
   }
   return replicaInfo;
 }
@@ -1737,23 +1736,37 @@ class FsDatasetImpl implements 
FsDatasetSpi {
   @Override // FsDatasetSpi
   public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
   throws IOException {
+ReplicaInfo replicaInfo = null;
+ReplicaInfo finalizedReplicaInfo = null;
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   if (Thread.interrupted()) {
 // Don't allow data modifications from interrupted threads
 throw new IOException("Cannot finalize block from Interrupted Thread");
   }
-  ReplicaInfo replicaInfo = getReplicaInfo(b);
+  replicaInfo = getReplicaInfo(b);
   if (replicaInfo.getState() == ReplicaState.FINALIZED) {
 // this is legal, when recovery happens on a file that has
 // been opened for append but never modified
 return;
   }
-  finalizeReplica(b.getBlockPoolId(), replicaInfo, fsyncDir);
+  finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo);
+}
+/*
+ * Sync the directory after rename from tmp/rbw to Finalized if
+ * configured. Though rename should be atomic operation, sync on both
+ * dest and src directories are done because IOUtils.fsync() calls
+ * directory's channel sync, not the journal itself.
+ */
+if (fsyncDir) {
+  FsVolumeSpi v = replicaInfo.getVolume();
+  File f = replicaInfo.getBlockFile();
+  File dest = finalizedReplicaInfo.getBlockFile();
+  fsyncDirectory(v, dest.getParentFile(), f.getParentFile());
 }
   }
 
-  private FinalizedReplica finalizeReplica(String bpid,
-  ReplicaInfo replicaInfo, boolean fsyncDir) throws IOException {
+  private FinalizedReplica finalizeReplica(String bpid, ReplicaInfo 
replicaInfo)
+  throws IOException {
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   FinalizedReplica newReplicaInfo = null;
   if (replicaInfo.getState() == ReplicaState.RUR &&
@@ -1773,15 +1786,6 @@ class FsDatasetImpl implements 
FsDatasetSpi {
 bpid, replicaInfo, f, replicaInfo.getBytesReserved());
 newReplicaInfo =
 new FinalizedReplica(replicaInfo,

hadoop git commit: HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by Vinayakumar B.

2017-08-09 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 4ed7df2c5 -> ae854076c


HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by 
Vinayakumar B.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae854076
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae854076
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae854076

Branch: refs/heads/branch-2.7
Commit: ae854076c743e674976e20c39bc8a9b9a75fa582
Parents: 4ed7df2
Author: Kihwal Lee 
Authored: Wed Aug 9 09:36:08 2017 -0500
Committer: Kihwal Lee 
Committed: Wed Aug 9 09:36:08 2017 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  2 +
 .../datanode/fsdataset/impl/FsDatasetImpl.java  | 61 ++--
 2 files changed, 34 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae854076/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 602c476..fe2d2b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -11,6 +11,8 @@ Release 2.7.5 - UNRELEASED
   OPTIMIZATIONS
 
   BUG FIXES
+HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock.
+(Vinayakumar B. via kihwal)
 
 Release 2.7.4 - 2017-08-04
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae854076/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index da8a628..07b7036 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -71,7 +71,6 @@ import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
-import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
@@ -928,8 +927,7 @@ class FsDatasetImpl implements FsDatasetSpi {
   targetVolume, blockFiles[0].getParentFile(), 0);
   newReplicaInfo.setNumBytes(blockFiles[1].length());
   // Finalize the copied files
-  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo,
-  false);
+  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
 
   removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
   oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
@@ -1254,7 +1252,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
 bumpReplicaGS(replicaInfo, newGS);
 // finalize the replica if RBW
 if (replicaInfo.getState() == ReplicaState.RBW) {
-  finalizeReplica(b.getBlockPoolId(), replicaInfo, false);
+  finalizeReplica(b.getBlockPoolId(), replicaInfo);
 }
 return replicaInfo;
   }
@@ -1583,23 +1581,38 @@ class FsDatasetImpl implements 
FsDatasetSpi {
* Complete the block write!
*/
   @Override // FsDatasetSpi
-  public synchronized void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
+  public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
   throws IOException {
-if (Thread.interrupted()) {
-  // Don't allow data modifications from interrupted threads
-  throw new IOException("Cannot finalize block from Interrupted Thread");
+ReplicaInfo replicaInfo = null;
+ReplicaInfo finalizedReplicaInfo = null;
+synchronized (this) {
+  if (Thread.interrupted()) {
+// Don't allow data modifications from interrupted threads
+throw new IOException("Cannot finalize block from Interrupted Thread");
+  }
+  replicaInfo = getReplicaInfo(b);
+  if (replicaInfo.getState() == ReplicaState.FINALIZED) {
+// this is legal, when recovery happens on a file that has
+// been opened for append but never modified
+return;
+  }
+  finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo);

hadoop git commit: HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by Vinayakumar B.

2017-08-09 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 49d459a3e -> 639380eff


HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by 
Vinayakumar B.

(cherry picked from commit 74bcc8d1f213ffd18056b7d0590d0cd346abaff9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/639380ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/639380ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/639380ef

Branch: refs/heads/branch-2.8
Commit: 639380efff495c17bb19f4e2657d1b51f1f04c4f
Parents: 49d459a
Author: Kihwal Lee 
Authored: Wed Aug 9 09:33:07 2017 -0500
Committer: Kihwal Lee 
Committed: Wed Aug 9 09:33:07 2017 -0500

--
 .../datanode/fsdataset/impl/FsDatasetImpl.java  | 40 +++-
 1 file changed, 22 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/639380ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 6468d65..ce0f7c0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1010,8 +1010,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   targetVolume, blockFiles[0].getParentFile(), 0);
   newReplicaInfo.setNumBytes(blockFiles[1].length());
   // Finalize the copied files
-  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo,
-  false);
+  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
   try(AutoCloseableLock lock = datasetLock.acquire()) {
 // Increment numBlocks here as this block moved without knowing to BPS
 FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume();
@@ -1372,7 +1371,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   bumpReplicaGS(replicaInfo, newGS);
   // finalize the replica if RBW
   if (replicaInfo.getState() == ReplicaState.RBW) {
-finalizeReplica(b.getBlockPoolId(), replicaInfo, false);
+finalizeReplica(b.getBlockPoolId(), replicaInfo);
   }
   return replicaInfo;
 }
@@ -1737,23 +1736,37 @@ class FsDatasetImpl implements 
FsDatasetSpi {
   @Override // FsDatasetSpi
   public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
   throws IOException {
+ReplicaInfo replicaInfo = null;
+ReplicaInfo finalizedReplicaInfo = null;
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   if (Thread.interrupted()) {
 // Don't allow data modifications from interrupted threads
 throw new IOException("Cannot finalize block from Interrupted Thread");
   }
-  ReplicaInfo replicaInfo = getReplicaInfo(b);
+  replicaInfo = getReplicaInfo(b);
   if (replicaInfo.getState() == ReplicaState.FINALIZED) {
 // this is legal, when recovery happens on a file that has
 // been opened for append but never modified
 return;
   }
-  finalizeReplica(b.getBlockPoolId(), replicaInfo, fsyncDir);
+  finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo);
+}
+/*
+ * Sync the directory after rename from tmp/rbw to Finalized if
+ * configured. Though rename should be atomic operation, sync on both
+ * dest and src directories are done because IOUtils.fsync() calls
+ * directory's channel sync, not the journal itself.
+ */
+if (fsyncDir) {
+  FsVolumeSpi v = replicaInfo.getVolume();
+  File f = replicaInfo.getBlockFile();
+  File dest = finalizedReplicaInfo.getBlockFile();
+  fsyncDirectory(v, dest.getParentFile(), f.getParentFile());
 }
   }
 
-  private FinalizedReplica finalizeReplica(String bpid,
-  ReplicaInfo replicaInfo, boolean fsyncDir) throws IOException {
+  private FinalizedReplica finalizeReplica(String bpid, ReplicaInfo 
replicaInfo)
+  throws IOException {
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   FinalizedReplica newReplicaInfo = null;
   if (replicaInfo.getState() == ReplicaState.RUR &&
@@ -1773,15 +1786,6 @@ class FsDatasetImpl implements 
FsDatasetSpi {
 bpid, replicaInfo, f, replicaInfo.getBytesReserved());
 newReplicaInfo =
 new FinalizedReplica(replicaInfo, v,

hadoop git commit: HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by Vinayakumar B.

2017-08-09 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 3e2753daa -> 74bcc8d1f


HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. Contributed by 
Vinayakumar B.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74bcc8d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74bcc8d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74bcc8d1

Branch: refs/heads/branch-2
Commit: 74bcc8d1f213ffd18056b7d0590d0cd346abaff9
Parents: 3e2753d
Author: Kihwal Lee 
Authored: Wed Aug 9 09:31:53 2017 -0500
Committer: Kihwal Lee 
Committed: Wed Aug 9 09:31:53 2017 -0500

--
 .../datanode/fsdataset/impl/FsDatasetImpl.java  | 40 +++-
 1 file changed, 22 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74bcc8d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 5a17bcd..c6cff69 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1035,8 +1035,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   targetVolume, blockFiles[0].getParentFile(), 0);
   newReplicaInfo.setNumBytes(blockFiles[1].length());
   // Finalize the copied files
-  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo,
-  false);
+  newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
   try(AutoCloseableLock lock = datasetLock.acquire()) {
 // Increment numBlocks here as this block moved without knowing to BPS
 FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume();
@@ -1409,7 +1408,7 @@ class FsDatasetImpl implements FsDatasetSpi 
{
   bumpReplicaGS(replicaInfo, newGS);
   // finalize the replica if RBW
   if (replicaInfo.getState() == ReplicaState.RBW) {
-finalizeReplica(b.getBlockPoolId(), replicaInfo, false);
+finalizeReplica(b.getBlockPoolId(), replicaInfo);
   }
   return replicaInfo;
 }
@@ -1777,23 +1776,37 @@ class FsDatasetImpl implements 
FsDatasetSpi {
   @Override // FsDatasetSpi
   public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
   throws IOException {
+ReplicaInfo replicaInfo = null;
+ReplicaInfo finalizedReplicaInfo = null;
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   if (Thread.interrupted()) {
 // Don't allow data modifications from interrupted threads
 throw new IOException("Cannot finalize block from Interrupted Thread");
   }
-  ReplicaInfo replicaInfo = getReplicaInfo(b);
+  replicaInfo = getReplicaInfo(b);
   if (replicaInfo.getState() == ReplicaState.FINALIZED) {
 // this is legal, when recovery happens on a file that has
 // been opened for append but never modified
 return;
   }
-  finalizeReplica(b.getBlockPoolId(), replicaInfo, fsyncDir);
+  finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo);
+}
+/*
+ * Sync the directory after rename from tmp/rbw to Finalized if
+ * configured. Though rename should be atomic operation, sync on both
+ * dest and src directories are done because IOUtils.fsync() calls
+ * directory's channel sync, not the journal itself.
+ */
+if (fsyncDir) {
+  FsVolumeSpi v = replicaInfo.getVolume();
+  File f = replicaInfo.getBlockFile();
+  File dest = finalizedReplicaInfo.getBlockFile();
+  fsyncDirectory(v, dest.getParentFile(), f.getParentFile());
 }
   }
 
-  private FinalizedReplica finalizeReplica(String bpid,
-  ReplicaInfo replicaInfo, boolean fsyncDir) throws IOException {
+  private FinalizedReplica finalizeReplica(String bpid, ReplicaInfo 
replicaInfo)
+  throws IOException {
 try(AutoCloseableLock lock = datasetLock.acquire()) {
   FinalizedReplica newReplicaInfo = null;
   if (replicaInfo.getState() == ReplicaState.RUR &&
@@ -1813,15 +1826,6 @@ class FsDatasetImpl implements 
FsDatasetSpi {
 bpid, replicaInfo, f, replicaInfo.getBytesReserved());
 newReplicaInfo =
 new FinalizedReplica(replicaInfo, v, dest.getParentFile());
-/*
- * Sync the directory after