This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5756478293 HDDS-9328. Speed up testListStatusIteratorOnPageSize (#5334)
5756478293 is described below
commit 57564782932a0889616b4ce001c1e083a198c740
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Oct 20 17:50:45 2023 +0200
HDDS-9328. Speed up testListStatusIteratorOnPageSize (#5334)
---
.../hadoop/fs/ozone/OzoneFileSystemTests.java | 100 +++++++++++++++++++++
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 59 +-----------
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 53 +----------
3 files changed, 104 insertions(+), 108 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
new file mode 100644
index 0000000000..64ce7c148c
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Common test cases for Ozone file systems.
+ */
+final class OzoneFileSystemTests {
+
+ private OzoneFileSystemTests() {
+ // no instances
+ }
+
+ /**
+ * Tests listStatusIterator operation on directory with different
+ * numbers of child directories.
+ */
+ public static void listStatusIteratorOnPageSize(OzoneConfiguration conf,
+ String rootPath) throws IOException {
+ final int pageSize = 32;
+ int[] dirCounts = {
+ 1,
+ pageSize - 1,
+ pageSize,
+ pageSize + 1,
+ pageSize + pageSize / 2,
+ pageSize + pageSize
+ };
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
+ URI uri = FileSystem.getDefaultUri(config);
+ config.setBoolean(
+ String.format("fs.%s.impl.disable.cache", uri.getScheme()), true);
+ FileSystem subject = FileSystem.get(uri, config);
+ Path dir = new Path(Objects.requireNonNull(rootPath),
"listStatusIterator");
+ try {
+ Set<String> paths = new TreeSet<>();
+ for (int dirCount : dirCounts) {
+ listStatusIterator(subject, dir, paths, dirCount);
+ }
+ } finally {
+ subject.delete(dir, true);
+ }
+ }
+
+ private static void listStatusIterator(FileSystem subject,
+ Path dir, Set<String> paths, int total) throws IOException {
+ for (int i = paths.size(); i < total; i++) {
+ Path p = new Path(dir, String.valueOf(i));
+ subject.mkdirs(p);
+ paths.add(p.getName());
+ }
+
+ RemoteIterator<FileStatus> iterator = subject.listStatusIterator(dir);
+ int iCount = 0;
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ FileStatus fileStatus = iterator.next();
+ iCount++;
+ String filename = fileStatus.getPath().getName();
+ assertTrue(filename + " not found", paths.contains(filename));
+ }
+ }
+
+ assertEquals(
+ "Total directories listed do not match the existing directories",
+ total, iCount);
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 3cab11714e..76a12204a6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -962,65 +962,10 @@ public class TestOzoneFileSystem {
}
}
- /**
- * Tests listStatusIterator operation on root directory with different
- * numbers of numDir.
- */
@Test
public void testListStatusIteratorOnPageSize() throws Exception {
- int[] pageSize = {
- 1, LISTING_PAGE_SIZE, LISTING_PAGE_SIZE + 1,
- LISTING_PAGE_SIZE - 1, LISTING_PAGE_SIZE + LISTING_PAGE_SIZE / 2,
- LISTING_PAGE_SIZE + LISTING_PAGE_SIZE
- };
- for (int numDir : pageSize) {
- int range = numDir / LISTING_PAGE_SIZE;
- switch (range) {
- case 0:
- listStatusIterator(numDir);
- break;
- case 1:
- listStatusIterator(numDir);
- break;
- case 2:
- listStatusIterator(numDir);
- break;
- default:
- listStatusIterator(numDir);
- }
- }
- }
-
- private void listStatusIterator(int numDirs) throws IOException {
- Path root = new Path("/" + volumeName + "/" + bucketName);
- Set<String> paths = new TreeSet<>();
- try {
- for (int i = 0; i < numDirs; i++) {
- Path p = new Path(root, String.valueOf(i));
- fs.mkdirs(p);
- paths.add(p.getName());
- }
-
- RemoteIterator<FileStatus> iterator = o3fs.listStatusIterator(root);
- int iCount = 0;
- if (iterator != null) {
- while (iterator.hasNext()) {
- FileStatus fileStatus = iterator.next();
- iCount++;
- Assert.assertTrue(paths.contains(fileStatus.getPath().getName()));
- }
- }
- Assert.assertEquals(
- "Total directories listed do not match the existing directories",
- numDirs, iCount);
-
- } finally {
- // Cleanup
- for (int i = 0; i < numDirs; i++) {
- Path p = new Path(root, String.valueOf(i));
- fs.delete(p, true);
- }
- }
+ OzoneFileSystemTests.listStatusIteratorOnPageSize(cluster.getConf(),
+ "/" + volumeName + "/" + bucketName);
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 07c9680aa4..65b8ebcc93 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -88,7 +88,6 @@ import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.security.PrivilegedExceptionAction;
@@ -119,7 +118,6 @@ import static
org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_OFS_SHARED_TMP_DIR;
@@ -588,55 +586,8 @@ public class TestRootedOzoneFileSystem {
*/
@Test
public void testListStatusIteratorOnPageSize() throws Exception {
- final int pageSize = 32;
- int[] dirCounts = {
- 1,
- pageSize - 1,
- pageSize,
- pageSize + 1,
- pageSize + pageSize / 2,
- pageSize + pageSize
- };
- OzoneConfiguration config = new OzoneConfiguration(conf);
- config.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
- URI uri = FileSystem.getDefaultUri(config);
- config.setBoolean(
- String.format("fs.%s.impl.disable.cache", uri.getScheme()), true);
- FileSystem subject = FileSystem.get(uri, config);
- Path root = new Path("/" + volumeName + "/" + bucketName);
- Path dir = new Path(root, "listStatusIterator");
- try {
- Set<String> paths = new TreeSet<>();
- for (int dirCount : dirCounts) {
- listStatusIterator(subject, dir, paths, dirCount);
- }
- } finally {
- subject.delete(dir, true);
- }
- }
-
- private static void listStatusIterator(FileSystem subject,
- Path dir, Set<String> paths, int total) throws IOException {
- for (int i = paths.size(); i < total; i++) {
- Path p = new Path(dir, String.valueOf(i));
- subject.mkdirs(p);
- paths.add(p.getName());
- }
-
- RemoteIterator<FileStatus> iterator = subject.listStatusIterator(dir);
- int iCount = 0;
- if (iterator != null) {
- while (iterator.hasNext()) {
- FileStatus fileStatus = iterator.next();
- iCount++;
- String filename = fileStatus.getPath().getName();
- assertTrue(filename + " not found", paths.contains(filename));
- }
- }
-
- assertEquals(
- "Total directories listed do not match the existing directories",
- total, iCount);
+ OzoneFileSystemTests.listStatusIteratorOnPageSize(conf,
+ "/" + volumeName + "/" + bucketName);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
