[06/14] hadoop git commit: HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang.

2018-04-13 Thread arp 
HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by 
HDFS-10997. Contributed by Yongjun Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56acbe47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56acbe47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56acbe47

Branch: refs/heads/branch-3.1
Commit: 56acbe476b6659ee4e3f53fdb5888906625950c8
Parents: 2b46bd3
Author: Yongjun Zhang 
Authored: Tue Mar 20 23:00:39 2018 -0700
Committer: Arpit Agarwal 
Committed: Fri Apr 13 10:14:24 2018 -0700

--
 .../hdfs/tools/snapshot/SnapshotDiff.java   |  1 +
 .../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 ++
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 41 
 .../snapshot/TestSnapshotDiffReport.java| 25 +---
 4 files changed, 80 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/56acbe47/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
index 3838ca1..1237099 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
@@ -106,6 +106,7 @@ public class SnapshotDiff extends Configured implements 
Tool {
 } catch (IOException e) {
   String[] content = e.getLocalizedMessage().split("\n");
   System.err.println("snapshotDiff: " + content[0]);
+  e.printStackTrace(System.err);
   return 1;
 }
 return 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56acbe47/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1411a7f..4f9f260 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -121,6 +121,9 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -2354,4 +2357,38 @@ public class DFSTestUtil {
 }
 return closedFiles;
   }
+
+  /**
+   * Check the correctness of the snapshotDiff report.
+   * Make sure all items in the passed entries are in the snapshotDiff
+   * report.
+   */
+  public static void verifySnapshotDiffReport(DistributedFileSystem fs,
+  Path dir, String from, String to,
+  DiffReportEntry... entries) throws IOException {
+SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to);
+// reverse the order of from and to
+SnapshotDiffReport inverseReport = fs
+.getSnapshotDiffReport(dir, to, from);
+LOG.info(report.toString());
+LOG.info(inverseReport.toString() + "\n");
+
+assertEquals(entries.length, report.getDiffList().size());
+assertEquals(entries.length, inverseReport.getDiffList().size());
+
+for (DiffReportEntry entry : entries) {
+  if (entry.getType() == DiffType.MODIFY) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(entry));
+  } else if (entry.getType() == DiffType.DELETE) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(
+new DiffReportEntry(DiffType.CREATE, entry.getSourcePath(;
+  } else if (entry.getType() == DiffType.CREATE) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(
+new DiffReportEntry(DiffType.DELETE, entry.getSourcePath(;
+  }
+}
+

hadoop git commit: HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang.

2018-03-21 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a3ed6f40e -> 6c63cc7d3


HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by 
HDFS-10997. Contributed by Yongjun Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c63cc7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c63cc7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c63cc7d

Branch: refs/heads/trunk
Commit: 6c63cc7d304571578e6551170552182d30b8e8fa
Parents: a3ed6f4
Author: Yongjun Zhang 
Authored: Tue Mar 20 23:00:39 2018 -0700
Committer: Yongjun Zhang 
Committed: Tue Mar 20 23:04:05 2018 -0700

--
 .../hdfs/tools/snapshot/SnapshotDiff.java   |  1 +
 .../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 ++
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 41 
 .../snapshot/TestSnapshotDiffReport.java| 25 +---
 4 files changed, 80 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
index 3838ca1..1237099 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
@@ -106,6 +106,7 @@ public class SnapshotDiff extends Configured implements 
Tool {
 } catch (IOException e) {
   String[] content = e.getLocalizedMessage().split("\n");
   System.err.println("snapshotDiff: " + content[0]);
+  e.printStackTrace(System.err);
   return 1;
 }
 return 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1411a7f..4f9f260 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -121,6 +121,9 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -2354,4 +2357,38 @@ public class DFSTestUtil {
 }
 return closedFiles;
   }
+
+  /**
+   * Check the correctness of the snapshotDiff report.
+   * Make sure all items in the passed entries are in the snapshotDiff
+   * report.
+   */
+  public static void verifySnapshotDiffReport(DistributedFileSystem fs,
+  Path dir, String from, String to,
+  DiffReportEntry... entries) throws IOException {
+SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to);
+// reverse the order of from and to
+SnapshotDiffReport inverseReport = fs
+.getSnapshotDiffReport(dir, to, from);
+LOG.info(report.toString());
+LOG.info(inverseReport.toString() + "\n");
+
+assertEquals(entries.length, report.getDiffList().size());
+assertEquals(entries.length, inverseReport.getDiffList().size());
+
+for (DiffReportEntry entry : entries) {
+  if (entry.getType() == DiffType.MODIFY) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(entry));
+  } else if (entry.getType() == DiffType.DELETE) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(
+new DiffReportEntry(DiffType.CREATE, entry.getSourcePath(;
+  } else if (entry.getType() == DiffType.CREATE) {
+assertTrue(report.getDiffList().contains(entry));
+assertTrue(inverseReport.getDiffList().contains(
+