Repository: hadoop Updated Branches: refs/heads/branch-2.7 ba0f66984 -> 42aa880df
HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode restart. Contributed by Xiaoyu Yao. (cherry picked from commit 3e715a4f4c46bcd8b3054cb0566e526c46bd5d66) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42aa880d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42aa880d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42aa880d Branch: refs/heads/branch-2.7 Commit: 42aa880df33497e53ad0cc31b1bb5c18a912798e Parents: ba0f669 Author: Kihwal Lee <kih...@apache.org> Authored: Fri Aug 28 13:27:17 2015 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Fri Aug 28 13:27:17 2015 -0500 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/namenode/FSImage.java | 6 ++---- .../hdfs/server/namenode/TestQuotaByStorageType.java | 14 ++++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42aa880d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 83f4fb5..31c6239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,5 +1,8 @@ Hadoop HDFS Change Log + HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode + restart. (xyao) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/42aa880d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 6eec9ee..5a7fbb1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -909,11 +909,9 @@ public class FSImage implements Closeable { + " quota = " + ssQuota + " < consumed = " + ssConsumed); } - final EnumCounters<StorageType> typeSpaces = - new EnumCounters<StorageType>(StorageType.class); + final EnumCounters<StorageType> typeSpaces = counts.getTypeSpaces(); for (StorageType t : StorageType.getTypesSupportingQuota()) { - final long typeSpace = counts.getTypeSpaces().get(t) - - parentTypeSpaces.get(t); + final long typeSpace = typeSpaces.get(t) - parentTypeSpaces.get(t); final long typeQuota = q.getTypeSpaces().get(t); if (Quota.isViolated(typeQuota, typeSpace)) { LOG.warn("Storage type quota violation in image for " http://git-wip-us.apache.org/repos/asf/hadoop/blob/42aa880d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java index 6d38937..ecbddac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java @@ -68,10 +68,7 @@ public class TestQuotaByStorageType { .storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT}) .build(); cluster.waitActive(); - - fsdir = cluster.getNamesystem().getFSDirectory(); - dfs = cluster.getFileSystem(); - fsn = cluster.getNamesystem(); + refreshClusterState(); } @After @@ -81,6 +78,13 @@ public class TestQuotaByStorageType { } } + // Cluster state must be refreshed after each start/restart in the test + private void refreshClusterState() throws IOException{ + fsdir = cluster.getNamesystem().getFSDirectory(); + dfs = cluster.getFileSystem(); + fsn = cluster.getNamesystem(); + } + @Test(timeout = 60000) public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception { testQuotaByStorageTypeWithFileCreateCase( @@ -663,6 +667,7 @@ public class TestQuotaByStorageType { // Restart namenode to make sure the editlog is correct cluster.restartNameNode(true); + refreshClusterState(); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); // Verify quota is still set @@ -715,6 +720,7 @@ public class TestQuotaByStorageType { dfs.saveNamespace(); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNode(true); + refreshClusterState(); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); assertTrue(testDirNode.isDirectory());
