Repository: hadoop Updated Branches: refs/heads/branch-2 65bfde552 -> bce3d442f refs/heads/trunk e2262d3d1 -> f2d7a67a2
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes non-snapshotted content. Contributed by Charles Lamb. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2d7a67a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2d7a67a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2d7a67a Branch: refs/heads/trunk Commit: f2d7a67a2c1d9dde10ed3171fdec65dff885afcc Parents: e2262d3 Author: Aaron T. Myers <a...@apache.org> Authored: Tue Mar 3 18:08:59 2015 -0800 Committer: Aaron T. Myers <a...@apache.org> Committed: Tue Mar 3 18:08:59 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +++ .../snapshot/TestSnapshotFileLength.java | 25 +++++++++++++++++--- 3 files changed, 28 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4e7b919..7ff3c78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1080,6 +1080,9 @@ Release 2.7.0 - UNRELEASED HDFS-6565. Use jackson instead jetty json in hdfs-client. (Akira Ajisaka via wheat9) + HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file + includes non-snapshotted content. (Charles Lamb via atm) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index abcd847..aac7b51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2220,6 +2220,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, // get block checksum for each block long remaining = length; + if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) { + remaining = Math.min(length, blockLocations.getFileLength()); + } for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) { if (refetchBlocks) { // refetch to get fresh tokens blockLocations = callGetBlockLocations(namenode, src, 0, length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java index 98aafc1..d53140f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java @@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot; import java.io.ByteArrayOutputStream; import java.io.PrintStream; - import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -29,8 +29,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -103,17 +104,35 @@ public class TestSnapshotFileLength { Path file1snap1 = SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name); + final FileChecksum snapChksum1 = hdfs.getFileChecksum(file1snap1); + assertThat("file and snapshot file checksums are not equal", + hdfs.getFileChecksum(file1), is(snapChksum1)); + // Append to the file. FSDataOutputStream out = hdfs.append(file1); + // Nothing has been appended yet. All checksums should still be equal. + assertThat("file and snapshot checksums (open for append) are not equal", + hdfs.getFileChecksum(file1), is(snapChksum1)); + assertThat("snapshot checksum (post-open for append) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); try { AppendTestUtil.write(out, 0, toAppend); // Test reading from snapshot of file that is open for append byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1); assertThat("Wrong data size in snapshot.", dataFromSnapshot.length, is(origLen)); + // Verify that checksum didn't change + assertThat("snapshot file checksum (pre-close) has changed", + hdfs.getFileChecksum(file1), is(snapChksum1)); + assertThat("snapshot checksum (post-append) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); } finally { out.close(); } + assertThat("file and snapshot file checksums (post-close) are equal", + hdfs.getFileChecksum(file1), not(snapChksum1)); + assertThat("snapshot file checksum (post-close) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.getFileStatus(file1);