hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. (cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)

2015-08-27 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.1 d3a7769bf - 6d2f80107


HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
(cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)

(cherry picked from commit 3827a1acdbc4f9fec3179dcafa614734b5fa31bc)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d2f8010
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d2f8010
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d2f8010

Branch: refs/heads/branch-2.6.1
Commit: 6d2f80107fa6d77f033ead6d5a1be190881df9df
Parents: d3a7769
Author: Tao Luo tao@wandisco.com
Authored: Wed Oct 29 20:20:11 2014 -0700
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Thu Aug 27 17:53:06 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  3 +-
 .../snapshot/TestSnapshotFileLength.java| 42 +++-
 3 files changed, 37 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d2f8010/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6bc6927..11d0038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -38,6 +38,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d2f8010/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index ff65ebc..db06d3b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -801,7 +801,8 @@ implements ByteBufferReadable, CanSetDropBehind, 
CanSetReadahead,
   }
   int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
   if (locatedBlocks.isLastBlockComplete()) {
-realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+realLen = (int) Math.min(realLen,
+locatedBlocks.getFileLength() - pos);
   }
   int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d2f8010/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 32534f0..98aafc1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -21,7 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
 
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,6 +58,8 @@ public class TestSnapshotFileLength {
 
   @Before
   public void setUp() throws Exception {
+conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCKSIZE);
+conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCKSIZE);
 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
   .build();
 cluster.waitActive();
@@ -81,40 +86,57 @@ public class TestSnapshotFileLength {
 
 int bytesRead;
 byte[] buffer = new byte[BLOCKSIZE * 8];
+int origLen = BLOCKSIZE + 1;
+int toAppend = BLOCKSIZE;
 FSDataInputStream fis = null;
 FileStatus fileStatus = null;
 
 // Create and write a file.
 Path

[27/50] [abbrv] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-24 Thread zhz 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa264114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa264114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa264114

Branch: refs/heads/HDFS-7285
Commit: fa2641143c0d74c4fef122d79f27791e15d3b43f
Parents: f2b4bc9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:45:43 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa264114/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e4e2896..1507cbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1819,9 +1819,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -2339,6 +2336,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

[29/50] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-17 Thread wangda 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa264114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa264114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa264114

Branch: refs/heads/YARN-1197
Commit: fa2641143c0d74c4fef122d79f27791e15d3b43f
Parents: f2b4bc9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:45:43 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa264114/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e4e2896..1507cbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1819,9 +1819,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -2339,6 +2336,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

[2/4] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-14 Thread vinayakumarb 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa264114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa264114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa264114

Branch: refs/heads/trunk
Commit: fa2641143c0d74c4fef122d79f27791e15d3b43f
Parents: f2b4bc9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:45:43 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa264114/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e4e2896..1507cbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1819,9 +1819,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -2339,6 +2336,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

[4/4] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-14 Thread vinayakumarb 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1

(cherry picked from commit fa2641143c0d74c4fef122d79f27791e15d3b43f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b06d3427
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b06d3427
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b06d3427

Branch: refs/heads/branch-2.7
Commit: b06d342749db39ec274d925dafc0627e891a1bee
Parents: f40714f
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:46:58 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06d3427/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a851e70..d06c368 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -688,9 +688,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -1209,6 +1206,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

[3/4] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-14 Thread vinayakumarb 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1

(cherry picked from commit fa2641143c0d74c4fef122d79f27791e15d3b43f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9839abe1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9839abe1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9839abe1

Branch: refs/heads/branch-2
Commit: 9839abe1a8f0f92b9fa37cef398b6ac09eb9389b
Parents: c2a9c39
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:46:30 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9839abe1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 869791c..7ae1b55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1486,9 +1486,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -2014,6 +2011,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

[1/4] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. (cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)

2015-08-14 Thread vinayakumarb 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c2a9c3929 - 9839abe1a
  refs/heads/branch-2.6 33e559d75 - 27991b6fd
  refs/heads/branch-2.7 f40714f8d - b06d34274
  refs/heads/trunk f2b4bc9b6 - fa2641143


HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
(cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)

Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27991b6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27991b6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27991b6f

Branch: refs/heads/branch-2.6
Commit: 27991b6fdb38931f9ce0f2c8a615f9fd9da2a02f
Parents: 33e559d
Author: Tao Luo tao@wandisco.com
Authored: Wed Oct 29 20:20:11 2014 -0700
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:43:31 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  3 +-
 .../snapshot/TestSnapshotFileLength.java| 42 +++-
 3 files changed, 37 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27991b6f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 10fd981..bbe7dba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -41,6 +41,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27991b6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index ff65ebc..db06d3b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -801,7 +801,8 @@ implements ByteBufferReadable, CanSetDropBehind, 
CanSetReadahead,
   }
   int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
   if (locatedBlocks.isLastBlockComplete()) {
-realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+realLen = (int) Math.min(realLen,
+locatedBlocks.getFileLength() - pos);
   }
   int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27991b6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 32534f0..98aafc1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -21,7 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
 
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,6 +58,8 @@ public class TestSnapshotFileLength {
 
   @Before
   public void setUp() throws Exception {
+conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCKSIZE);
+conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCKSIZE);
 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
   .build();
 cluster.waitActive();
@@ -81,40 +86,57 @@ public class TestSnapshotFileLength {
 
 int bytesRead;
 byte[] buffer = new byte[BLOCKSIZE * 8];
+int origLen = BLOCKSIZE + 1;
+int toAppend = BLOCKSIZE;

[03/43] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. (cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)

2015-08-14 Thread sjlee 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
(cherry picked from commit 8bfef590295372a48bd447b1462048008810ee17)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3827a1ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3827a1ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3827a1ac

Branch: refs/heads/sjlee/hdfs-merge
Commit: 3827a1acdbc4f9fec3179dcafa614734b5fa31bc
Parents: 1aa9e34
Author: Tao Luo tao@wandisco.com
Authored: Wed Oct 29 20:20:11 2014 -0700
Committer: Sangjin Lee sj...@apache.org
Committed: Wed Aug 12 21:25:12 2015 -0700

--
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  3 +-
 .../snapshot/TestSnapshotFileLength.java| 42 +++-
 2 files changed, 34 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3827a1ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index ff65ebc..db06d3b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -801,7 +801,8 @@ implements ByteBufferReadable, CanSetDropBehind, 
CanSetReadahead,
   }
   int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
   if (locatedBlocks.isLastBlockComplete()) {
-realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+realLen = (int) Math.min(realLen,
+locatedBlocks.getFileLength() - pos);
   }
   int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3827a1ac/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 32534f0..98aafc1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -21,7 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
 
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,6 +58,8 @@ public class TestSnapshotFileLength {
 
   @Before
   public void setUp() throws Exception {
+conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCKSIZE);
+conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCKSIZE);
 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
   .build();
 cluster.waitActive();
@@ -81,40 +86,57 @@ public class TestSnapshotFileLength {
 
 int bytesRead;
 byte[] buffer = new byte[BLOCKSIZE * 8];
+int origLen = BLOCKSIZE + 1;
+int toAppend = BLOCKSIZE;
 FSDataInputStream fis = null;
 FileStatus fileStatus = null;
 
 // Create and write a file.
 Path file1 = new Path(sub, file1Name);
-DFSTestUtil.createFile(hdfs, file1, 0, REPLICATION, SEED);
-DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE);
+DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, 0, BLOCKSIZE, REPLICATION, 
SEED);
+DFSTestUtil.appendFile(hdfs, file1, origLen);
 
 // Create a snapshot on the parent directory.
 hdfs.allowSnapshot(sub);
 hdfs.createSnapshot(sub, snapshot1);
 
-// Write more data to the file.
-DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE);
+Path file1snap1
+= SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name);
+
+// Append to the file.
+FSDataOutputStream out = hdfs.append(file1);
+try {
+  AppendTestUtil.write(out, 0, toAppend);
+  // Test reading from snapshot of file that is open for append
+  byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1);
+  assertThat(Wrong data size in snapshot.,
+

[09/36] hadoop git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo. Moved CHANGES.txt entry to 2.6.1

2015-08-14 Thread zhz 
HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.
Moved CHANGES.txt entry to 2.6.1


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa264114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa264114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa264114

Branch: refs/heads/HDFS-7285-merge
Commit: fa2641143c0d74c4fef122d79f27791e15d3b43f
Parents: f2b4bc9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Aug 14 11:45:43 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Aug 14 11:45:43 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa264114/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e4e2896..1507cbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1819,9 +1819,6 @@ Release 2.7.0 - 2015-04-20
 HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
 (Zhe Zhang via wheat9)
 
-HDFS-7263. Snapshot read can reveal future bytes for appended files.
-(Tao Luo via shv)
-
 HDFS-7315. DFSTestUtil.readFileBuffer opens extra FSDataInputStream.
 (Plamen Jeliazkov via wheat9)
 
@@ -2339,6 +2336,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo.

2014-10-29 Thread shv 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2a6be65c8 - 0126cf16b


HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0126cf16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0126cf16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0126cf16

Branch: refs/heads/trunk
Commit: 0126cf16b73843da2e504b6a03fee8bd93a404d5
Parents: 2a6be65
Author: Tao Luo tao@wandisco.com
Authored: Wed Oct 29 20:15:01 2014 -0700
Committer: Konstantin V Shvachko s...@apache.org
Committed: Wed Oct 29 20:21:12 2014 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  2 +
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  3 +-
 .../snapshot/TestSnapshotFileLength.java| 42 +++-
 3 files changed, 36 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0126cf16/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 11f757a..90c4b3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -367,6 +367,8 @@ Release 2.7.0 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
 
 Release 2.6.0 - UNRELEASED
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0126cf16/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index e83f067..008cc54 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -804,7 +804,8 @@ implements ByteBufferReadable, CanSetDropBehind, 
CanSetReadahead,
   }
   int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
   if (locatedBlocks.isLastBlockComplete()) {
-realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+realLen = (int) Math.min(realLen,
+locatedBlocks.getFileLength() - pos);
   }
   int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0126cf16/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 32534f0..98aafc1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -21,7 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
 
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,6 +58,8 @@ public class TestSnapshotFileLength {
 
   @Before
   public void setUp() throws Exception {
+conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCKSIZE);
+conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCKSIZE);
 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
   .build();
 cluster.waitActive();
@@ -81,40 +86,57 @@ public class TestSnapshotFileLength {
 
 int bytesRead;
 byte[] buffer = new byte[BLOCKSIZE * 8];
+int origLen = BLOCKSIZE + 1;
+int toAppend = BLOCKSIZE;
 FSDataInputStream fis = null;
 FileStatus fileStatus = null;
 
 // Create and write a file.
 Path file1 = new Path(sub, file1Name);
-DFSTestUtil.createFile(hdfs, file1, 0, REPLICATION, SEED);
-DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE);
+

git commit: HDFS-7263. Snapshot read can reveal future bytes for appended files. Contributed by Tao Luo.

2014-10-29 Thread shv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 51f621c01 - 8bfef5902


HDFS-7263. Snapshot read can reveal future bytes for appended files. 
Contributed by Tao Luo.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bfef590
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bfef590
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bfef590

Branch: refs/heads/branch-2
Commit: 8bfef590295372a48bd447b1462048008810ee17
Parents: 51f621c
Author: Tao Luo tao@wandisco.com
Authored: Wed Oct 29 20:20:11 2014 -0700
Committer: Konstantin V Shvachko s...@apache.org
Committed: Wed Oct 29 20:20:11 2014 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |  3 +-
 .../snapshot/TestSnapshotFileLength.java| 42 +++-
 3 files changed, 37 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfef590/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 566a219..18d569f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -114,6 +114,9 @@ Release 2.7.0 - UNRELEASED
 HDFS-7235. DataNode#transferBlock should report blocks that don't exist
 using reportBadBlock (yzhang via cmccabe)
 
+HDFS-7263. Snapshot read can reveal future bytes for appended files.
+(Tao Luo via shv)
+
 
 Release 2.6.0 - UNRELEASED
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfef590/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index e83f067..008cc54 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -804,7 +804,8 @@ implements ByteBufferReadable, CanSetDropBehind, 
CanSetReadahead,
   }
   int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
   if (locatedBlocks.isLastBlockComplete()) {
-realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+realLen = (int) Math.min(realLen,
+locatedBlocks.getFileLength() - pos);
   }
   int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfef590/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 32534f0..98aafc1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -21,7 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
 
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,6 +58,8 @@ public class TestSnapshotFileLength {
 
   @Before
   public void setUp() throws Exception {
+conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCKSIZE);
+conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCKSIZE);
 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
   .build();
 cluster.waitActive();
@@ -81,40 +86,57 @@ public class TestSnapshotFileLength {
 
 int bytesRead;
 byte[] buffer = new byte[BLOCKSIZE * 8];
+int origLen = BLOCKSIZE + 1;
+int toAppend = BLOCKSIZE;
 FSDataInputStream fis = null;
 FileStatus fileStatus = null;
 
 // Create and write a file.
 Path file1 = new Path(sub, file1Name);
-DFSTestUtil.createFile(hdfs, file1, 0, REPLICATION, SEED);
-DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE);
+