This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-2665-ofs in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push: new 28540cb HDDS-3574. Implement ofs://: Override getTrashRoot (#941) 28540cb is described below commit 28540cb1cc1f27c43f727b797b408125e21a9abb Author: Siyao Meng <50227127+smen...@users.noreply.github.com> AuthorDate: Tue Jun 2 07:53:21 2020 -0700 HDDS-3574. Implement ofs://: Override getTrashRoot (#941) --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 88 ++++++++++++++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +++ .../java/org/apache/hadoop/fs/ozone/OFSPath.java | 130 ++++++++++++++++----- .../org/apache/hadoop/fs/ozone/TestOFSPath.java | 36 ++++-- 4 files changed, 228 insertions(+), 39 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index caf9c69..6e1540c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -18,7 +18,12 @@ package org.apache.hadoop.ozone.shell; import com.google.common.base.Strings; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.ozone.OFSPath; +import org.apache.hadoop.fs.ozone.OzoneFsShell; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; @@ -29,6 +34,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.shell.s3.S3Shell; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -48,11 +54,15 @@ import picocli.CommandLine.RunLast; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.PrintStream; import java.util.Arrays; import java.util.List; import java.util.UUID; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.junit.Assert.fail; /** @@ -442,6 +452,84 @@ public class TestOzoneShellHA { Assert.assertEquals(0, getNumOfBuckets("bucket")); } + /** + * Helper function to retrieve Ozone client configuration for trash testing. + * @param hostPrefix Scheme + Authority. e.g. ofs://om-service-test1 + * @param configuration Server config to generate client config from. + * @return Config added with fs.ofs.impl, fs.defaultFS and fs.trash.interval. + */ + private OzoneConfiguration getClientConfForOFS( + String hostPrefix, OzoneConfiguration configuration) { + + OzoneConfiguration clientConf = new OzoneConfiguration(configuration); + clientConf.set("fs.ofs.impl", + "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem"); + clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix); + clientConf.setInt(FS_TRASH_INTERVAL_KEY, 60); + return clientConf; + } + + @Test + public void testDeleteToTrashOrSkipTrash() throws Exception { + final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId; + OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf); + OzoneFsShell shell = new OzoneFsShell(clientConf); + FileSystem fs = FileSystem.get(clientConf); + final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1"; + // Note: CURRENT is also privately defined in TrashPolicyDefault + final Path trashCurrent = new Path("Current"); + + final String strKey1 = strDir1 + "/key1"; + final Path pathKey1 = new Path(strKey1); + final Path trashPathKey1 = Path.mergePaths(new Path( + new OFSPath(strKey1).getTrashRoot(), trashCurrent), pathKey1); + + final String strKey2 = strDir1 + "/key2"; + final Path pathKey2 = new Path(strKey2); + final Path trashPathKey2 = Path.mergePaths(new Path( + new OFSPath(strKey2).getTrashRoot(), trashCurrent), pathKey2); + + int res; + try { + res = ToolRunner.run(shell, new String[]{"-mkdir", "-p", strDir1}); + Assert.assertEquals(0, res); + + // Check delete to trash behavior + res = ToolRunner.run(shell, new String[]{"-touch", strKey1}); + Assert.assertEquals(0, res); + // Verify key1 creation + FileStatus statusPathKey1 = fs.getFileStatus(pathKey1); + Assert.assertEquals(strKey1, statusPathKey1.getPath().toString()); + // rm without -skipTrash. since trash interval > 0, should moved to trash + res = ToolRunner.run(shell, new String[]{"-rm", strKey1}); + Assert.assertEquals(0, res); + // Verify that the file is moved to the correct trash location + FileStatus statusTrashPathKey1 = fs.getFileStatus(trashPathKey1); + // It'd be more meaningful if we actually write some content to the file + Assert.assertEquals( + statusPathKey1.getLen(), statusTrashPathKey1.getLen()); + Assert.assertEquals( + fs.getFileChecksum(pathKey1), fs.getFileChecksum(trashPathKey1)); + + // Check delete skip trash behavior + res = ToolRunner.run(shell, new String[]{"-touch", strKey2}); + Assert.assertEquals(0, res); + // Verify key2 creation + FileStatus statusPathKey2 = fs.getFileStatus(pathKey2); + Assert.assertEquals(strKey2, statusPathKey2.getPath().toString()); + // rm with -skipTrash + res = ToolRunner.run(shell, new String[]{"-rm", "-skipTrash", strKey2}); + Assert.assertEquals(0, res); + // Verify that the file is NOT moved to the trash location + try { + fs.getFileStatus(trashPathKey2); + Assert.fail("getFileStatus on non-existent should throw."); + } catch (FileNotFoundException ignored) { + } + } finally { + shell.close(); + } + } @Test public void testS3PathCommand() throws Exception { diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index fd6df55..dbf0074 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -659,6 +659,19 @@ public class BasicRootedOzoneFileSystem extends FileSystem { } /** + * Get the root directory of Trash for a path in OFS. + * Returns /<volumename>/<bucketname>/.Trash/<username> + * Caller appends either Current or checkpoint timestamp for trash destination + * @param path the trash root of the path to be determined. + * @return trash root + */ + @Override + public Path getTrashRoot(Path path) { + OFSPath ofsPath = new OFSPath(path); + return ofsPath.getTrashRoot(); + } + + /** * Creates a directory. Directory is represented using a key with no value. * * @param path directory path to be created diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java index 88f89fc..f602833 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java @@ -34,6 +34,8 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.StringTokenizer; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; /** @@ -41,18 +43,19 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; */ @InterfaceAudience.Private @InterfaceStability.Unstable -class OFSPath { +public class OFSPath { + private String authority = ""; /** * Here is a table illustrating what each name variable is given an input path * Assuming /tmp is mounted to /tempVol/tempBucket * (empty) = empty string "". * - * Path volumeName bucketName mountName keyName + * Path volumeName bucketName mountName keyName * -------------------------------------------------------------------------- - * /vol1/buc2/dir3/key4 vol1 buc2 (empty) dir3/key4 - * /vol1/buc2 vol1 buc2 (empty) (empty) - * /vol1 vol1 (empty) (empty) (empty) - * /tmp/dir3/key4 tmp <username> tmp dir3/key4 + * /vol1/buc2/dir3/key4 vol1 buc2 (empty) dir3/key4 + * /vol1/buc2 vol1 buc2 (empty) (empty) + * /vol1 vol1 (empty) (empty) (empty) + * /tmp/dir3/key4 tmp md5(<username>) tmp dir3/key4 * * Note the leading '/' doesn't matter. */ @@ -65,37 +68,38 @@ class OFSPath { @VisibleForTesting static final String OFS_MOUNT_TMP_VOLUMENAME = "tmp"; - OFSPath(Path path) { - String pathStr = path.toUri().getPath(); - initOFSPath(pathStr); + public OFSPath(Path path) { + initOFSPath(path.toUri()); } - OFSPath(String pathStr) { - initOFSPath(pathStr); + public OFSPath(String pathStr) { + try { + initOFSPath(new URI(pathStr)); + } catch (URISyntaxException ex) { + throw new RuntimeException(ex); + } } - private void initOFSPath(String pathStr) { - // pathStr should not have authority - try { - URI uri = new URI(pathStr); - String authority = uri.getAuthority(); - if (authority != null && !authority.isEmpty()) { - throw new ParseException("Invalid path " + pathStr + - ". Shouldn't contain authority."); + private void initOFSPath(URI uri) { + // Scheme is case-insensitive + String scheme = uri.getScheme(); + if (scheme != null) { + if (!scheme.toLowerCase().equals(OZONE_OFS_URI_SCHEME)) { + throw new ParseException("Can't parse schemes other than ofs://."); } - } catch (URISyntaxException ex) { - throw new ParseException("Failed to parse path " + pathStr + " as URI."); } - // tokenize + // authority could be empty + authority = uri.getAuthority() == null ? "" : uri.getAuthority(); + String pathStr = uri.getPath(); StringTokenizer token = new StringTokenizer(pathStr, OZONE_URI_DELIMITER); int numToken = token.countTokens(); + if (numToken > 0) { String firstToken = token.nextToken(); - // TODO: Compare a keyword list instead for future expansion. + // TODO: Compare a list of mounts in the future. if (firstToken.equals(OFS_MOUNT_NAME_TMP)) { mountName = firstToken; - // TODO: In the future, may retrieve volume and bucket from - // UserVolumeInfo on the server side. TBD. + // TODO: Make this configurable in the future. volumeName = OFS_MOUNT_TMP_VOLUMENAME; try { bucketName = getTempMountBucketNameOfCurrentUser(); @@ -119,6 +123,10 @@ class OFSPath { } } + public String getAuthority() { + return authority; + } + public String getVolumeName() { return volumeName; } @@ -137,6 +145,39 @@ class OFSPath { } /** + * Return the reconstructed path string. + * Directories including volumes and buckets will have a trailing '/'. + */ + @Override + public String toString() { + Preconditions.checkNotNull(authority); + StringBuilder sb = new StringBuilder(); + if (!isMount()) { + sb.append(volumeName); + sb.append(OZONE_URI_DELIMITER); + if (!bucketName.isEmpty()) { + sb.append(bucketName); + sb.append(OZONE_URI_DELIMITER); + } + } else { + sb.append(mountName); + sb.append(OZONE_URI_DELIMITER); + } + if (!keyName.isEmpty()) { + sb.append(keyName); + } + if (authority.isEmpty()) { + sb.insert(0, OZONE_URI_DELIMITER); + return sb.toString(); + } else { + final Path pathWithSchemeAuthority = new Path( + OZONE_OFS_URI_SCHEME, authority, OZONE_URI_DELIMITER); + sb.insert(0, pathWithSchemeAuthority.toString()); + return sb.toString(); + } + } + + /** * Get the volume & bucket or mount name (non-key path). * @return String of path excluding key in bucket. */ @@ -176,15 +217,15 @@ class OFSPath { /** * If both volume and bucket names are empty, the given path is root. - * i.e. / + * i.e. / is root. */ public boolean isRoot() { return this.getVolumeName().isEmpty() && this.getBucketName().isEmpty(); } /** - * If bucket name is empty but volume name is not, the given path is volume. - * e.g. /volume1 + * If bucket name is empty but volume name is not, the given path is a volume. + * e.g. /volume1 is a volume. */ public boolean isVolume() { return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty(); @@ -192,8 +233,8 @@ class OFSPath { /** * If key name is empty but volume and bucket names are not, the given path - * it bucket. - * e.g. /volume1/bucket2 + * is a bucket. + * e.g. /volume1/bucket2 is a bucket. */ public boolean isBucket() { return this.getKeyName().isEmpty() && @@ -201,6 +242,14 @@ class OFSPath { !this.getVolumeName().isEmpty(); } + /** + * If key name is not empty, the given path is a key. + * e.g. /volume1/bucket2/key3 is a key. + */ + public boolean isKey() { + return !this.getKeyName().isEmpty(); + } + private static String md5Hex(String input) { try { MessageDigest md = MessageDigest.getInstance("MD5"); @@ -238,4 +287,25 @@ class OFSPath { String username = UserGroupInformation.getCurrentUser().getUserName(); return getTempMountBucketName(username); } + + /** + * Return trash root for the given path. + * @return trash root for the given path. + */ + public Path getTrashRoot() { + if (!this.isKey()) { + throw new RuntimeException("Volume or bucket doesn't have trash root."); + } + try { + String username = UserGroupInformation.getCurrentUser().getUserName(); + final Path pathRoot = new Path( + OZONE_OFS_URI_SCHEME, authority, OZONE_URI_DELIMITER); + final Path pathToVolume = new Path(pathRoot, volumeName); + final Path pathToBucket = new Path(pathToVolume, bucketName); + final Path pathToTrash = new Path(pathToBucket, TRASH_PREFIX); + return new Path(pathToTrash, username); + } catch (IOException ex) { + throw new RuntimeException("getTrashRoot failed.", ex); + } + } } diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java index c46c09f..afdeb51 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java @@ -31,56 +31,67 @@ public class TestOFSPath { public void testParsingVolumeBucketWithKey() { // Two most common cases: file key and dir key inside a bucket OFSPath ofsPath = new OFSPath("/volume1/bucket2/dir3/key4"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("bucket2", ofsPath.getBucketName()); Assert.assertEquals("dir3/key4", ofsPath.getKeyName()); Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/bucket2/dir3/key4", ofsPath.toString()); // The ending '/' matters for key inside a bucket, indicating directory ofsPath = new OFSPath("/volume1/bucket2/dir3/dir5/"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("bucket2", ofsPath.getBucketName()); // Check the key must end with '/' (dir5 is a directory) Assert.assertEquals("dir3/dir5/", ofsPath.getKeyName()); Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/bucket2/dir3/dir5/", ofsPath.toString()); } @Test public void testParsingVolumeBucketOnly() { // Volume and bucket only OFSPath ofsPath = new OFSPath("/volume1/bucket2/"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("bucket2", ofsPath.getBucketName()); Assert.assertEquals("", ofsPath.getMountName()); Assert.assertEquals("", ofsPath.getKeyName()); Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/bucket2/", ofsPath.toString()); - // The ending '/' shouldn't for buckets + // The trailing '/' doesn't matter when parsing a bucket path ofsPath = new OFSPath("/volume1/bucket2"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("bucket2", ofsPath.getBucketName()); Assert.assertEquals("", ofsPath.getMountName()); Assert.assertEquals("", ofsPath.getKeyName()); Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/bucket2/", ofsPath.toString()); } @Test public void testParsingVolumeOnly() { // Volume only OFSPath ofsPath = new OFSPath("/volume1/"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("", ofsPath.getBucketName()); Assert.assertEquals("", ofsPath.getMountName()); Assert.assertEquals("", ofsPath.getKeyName()); Assert.assertEquals("/volume1/", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/", ofsPath.toString()); - // Ending '/' shouldn't matter + // The trailing '/' doesn't matter when parsing a volume path ofsPath = new OFSPath("/volume1"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals("volume1", ofsPath.getVolumeName()); Assert.assertEquals("", ofsPath.getBucketName()); Assert.assertEquals("", ofsPath.getMountName()); @@ -90,17 +101,20 @@ public class TestOFSPath { // The behavior might change in the future. Assert.assertEquals("/volume1/", ofsPath.getNonKeyPath()); Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("/volume1/", ofsPath.toString()); } @Test public void testParsingWithAuthority() { - try { - new OFSPath("ofs://svc1/volume1/bucket1/dir1/"); - Assert.fail( - "Should have thrown exception when parsing path with authority."); - } catch (Exception ignored) { - // Test pass - } + OFSPath ofsPath = new OFSPath("ofs://svc1:9876/volume1/bucket2/dir3/"); + Assert.assertEquals("svc1:9876", ofsPath.getAuthority()); + Assert.assertEquals("volume1", ofsPath.getVolumeName()); + Assert.assertEquals("bucket2", ofsPath.getBucketName()); + Assert.assertEquals("dir3/", ofsPath.getKeyName()); + Assert.assertEquals("/volume1/bucket2", ofsPath.getNonKeyPath()); + Assert.assertFalse(ofsPath.isMount()); + Assert.assertEquals("ofs://svc1:9876/volume1/bucket2/dir3/", + ofsPath.toString()); } @Test @@ -115,6 +129,7 @@ public class TestOFSPath { } // Mount only OFSPath ofsPath = new OFSPath("/tmp/"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals( OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName()); Assert.assertEquals(bucketName, ofsPath.getBucketName()); @@ -122,9 +137,11 @@ public class TestOFSPath { Assert.assertEquals("", ofsPath.getKeyName()); Assert.assertEquals("/tmp", ofsPath.getNonKeyPath()); Assert.assertTrue(ofsPath.isMount()); + Assert.assertEquals("/tmp/", ofsPath.toString()); // Mount with key ofsPath = new OFSPath("/tmp/key1"); + Assert.assertEquals("", ofsPath.getAuthority()); Assert.assertEquals( OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName()); Assert.assertEquals(bucketName, ofsPath.getBucketName()); @@ -132,5 +149,6 @@ public class TestOFSPath { Assert.assertEquals("key1", ofsPath.getKeyName()); Assert.assertEquals("/tmp", ofsPath.getNonKeyPath()); Assert.assertTrue(ofsPath.isMount()); + Assert.assertEquals("/tmp/key1", ofsPath.toString()); } } --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org