This is an automated email from the ASF dual-hosted git repository.
xyao pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push:
new 12ec7c3 HDFS-14633. The StorageType quota and consume in QuotaFeature
is not handled for rename. Contributed by Jinglun.
12ec7c3 is described below
commit 12ec7c301ff19b004fd94e3f0af7238f8050bccc
Author: Xiaoyu Yao <x...@apache.org>
AuthorDate: Fri Aug 30 16:46:04 2019 -0700
HDFS-14633. The StorageType quota and consume in QuotaFeature is not
handled for rename. Contributed by Jinglun.
(cherry picked from commit 62d71fbac3789c7d484bc76ced9ec7fa6ff94de1)
---
.../hadoop/hdfs/server/namenode/FSDirRenameOp.java | 7 ++-
.../hadoop/hdfs/server/namenode/FSDirectory.java | 5 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 63 +++++++++++++++++++++-
3 files changed, 72 insertions(+), 3 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index e6f1dd1..23e0d16 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -78,7 +78,12 @@ class FSDirRenameOp {
while(src.getINode(i) == dst.getINode(i)) { i++; }
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
- final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);
+ // Assume dstParent existence check done by callers.
+ INode dstParent = dst.getINode(-2);
+ // Use the destination parent's storage policy for quota delta verify.
+ final QuotaCounts delta = src.getLastINode()
+ .computeQuotaUsage(bsps, dstParent.getStoragePolicyID(), false,
+ Snapshot.CURRENT_STATE_ID);
// Reduce the required quota by dst that is being removed
final INode dstINode = dst.getLastINode();
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 437d359..31b9cce 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -52,6 +52,7 @@ import
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import
org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo.UpdatedReplicationInfo;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -1106,7 +1107,9 @@ public class FSDirectory implements Closeable {
// always verify inode name
verifyINodeName(inode.getLocalNameBytes());
- final QuotaCounts counts =
inode.computeQuotaUsage(getBlockStoragePolicySuite());
+ final QuotaCounts counts = inode
+ .computeQuotaUsage(getBlockStoragePolicySuite(),
+ parent.getStoragePolicyID(), false, Snapshot.CURRENT_STATE_ID);
updateCount(existing, pos, counts, checkQuota);
boolean isRename = (inode.getParent() != null);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index c46620d..cdd787a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -1539,6 +1538,57 @@ public class TestQuota {
}
@Test
+ public void testClrQuotaOnRoot() throws Exception {
+ long orignalQuota = dfs.getQuotaUsage(new Path("/")).getQuota();
+ DFSAdmin admin = new DFSAdmin(conf);
+ String[] args;
+ args = new String[] {"-setQuota", "3K", "/"};
+ runCommand(admin, args, false);
+ assertEquals(3 * 1024, dfs.getQuotaUsage(new Path("/")).getQuota());
+ args = new String[] {"-clrQuota", "/"};
+ runCommand(admin, args, false);
+ assertEquals(orignalQuota, dfs.getQuotaUsage(new Path("/")).getQuota());
+ }
+
+ @Test
+ public void testRename() throws Exception {
+ int fileLen = 1024;
+ short replication = 3;
+
+ final Path parent = new Path(PathUtils.getTestDir(getClass()).getPath(),
+ GenericTestUtils.getMethodName());
+ assertTrue(dfs.mkdirs(parent));
+
+ final Path srcDir = new Path(parent, "src-dir");
+ Path file = new Path(srcDir, "file1");
+ DFSTestUtil.createFile(dfs, file, fileLen, replication, 0);
+ dfs.setStoragePolicy(srcDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+
+ final Path dstDir = new Path(parent, "dst-dir");
+ assertTrue(dfs.mkdirs(dstDir));
+ dfs.setStoragePolicy(dstDir, HdfsConstants.ALLSSD_STORAGE_POLICY_NAME);
+
+ dfs.setQuota(srcDir, 100000, 100000);
+ dfs.setQuota(dstDir, 100000, 100000);
+
+ Path dstFile = new Path(dstDir, "file1");
+ // Test quota check of rename. Expect a QuotaExceedException.
+ dfs.setQuotaByStorageType(dstDir, StorageType.SSD, 10);
+ try {
+ dfs.rename(file, dstFile);
+ fail("Expect QuotaExceedException.");
+ } catch (QuotaExceededException qe) {
+ }
+
+ // Set enough quota, expect a successful rename.
+ dfs.setQuotaByStorageType(dstDir, StorageType.SSD, fileLen * replication);
+ dfs.rename(file, dstFile);
+ // Verify the storage type usage is properly updated on source and dst.
+ checkQuotaAndCount(dfs, srcDir);
+ checkQuotaAndCount(dfs, dstDir);
+ }
+
+ @Test
public void testSpaceQuotaExceptionOnAppend() throws Exception {
GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.TRACE);
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.TRACE);
@@ -1606,4 +1656,15 @@ public class TestQuota {
}
scanner.close();
}
+
+ // quota and count should match.
+ private void checkQuotaAndCount(DistributedFileSystem fs, Path path)
+ throws IOException {
+ QuotaUsage qu = fs.getQuotaUsage(path);
+ ContentSummary cs = fs.getContentSummary(path);
+ for (StorageType st : StorageType.values()) {
+ // it will fail here, because the quota and consume is not handled right.
+ assertEquals(qu.getTypeConsumed(st), cs.getTypeConsumed(st));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
