This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new edcb2ac2a5 HDDS-10835. Show overwritten hsync keys in ListOpenFile CLI
(#6661)
edcb2ac2a5 is described below
commit edcb2ac2a5afe970c47273ed84cd7a887a850034
Author: Sammi Chen <[email protected]>
AuthorDate: Tue May 14 07:15:29 2024 +0800
HDDS-10835. Show overwritten hsync keys in ListOpenFile CLI (#6661)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 127 +++++++++++++++++++++
.../ozone/admin/om/ListOpenFilesSubCommand.java | 27 ++++-
2 files changed, 150 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index aa4d851678..87c72747aa 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -819,6 +819,133 @@ public class TestOzoneShellHA {
}
}
+ @Test
+ public void testAdminCmdListOpenFilesWithOverwrittenKeys()
+ throws Exception {
+
+ OzoneConfiguration conf = cluster.getConf();
+ final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
+
+ OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ clientConf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
+ FileSystem fs = FileSystem.get(clientConf);
+
+ assertNotEquals(fs.getConf().get(OZONE_FS_HSYNC_ENABLED),
+ "false", OZONE_FS_HSYNC_ENABLED + " is set to false " +
+ "by external force. Must be true to allow hsync to function");
+
+ final String volumeName = "volume-list-del";
+ final String bucketName = "buck1";
+
+ String dir1 = hostPrefix +
+ OM_KEY_PREFIX + volumeName +
+ OM_KEY_PREFIX + bucketName +
+ OM_KEY_PREFIX + "dir1";
+ // Create volume, bucket, dir
+ assertTrue(fs.mkdirs(new Path(dir1)));
+ String keyPrefix = OM_KEY_PREFIX + "key";
+
+ final int numKeys = 5;
+ String[] keys = new String[numKeys];
+
+ for (int i = 0; i < numKeys; i++) {
+ keys[i] = dir1 + keyPrefix + i;
+ }
+
+ String pathToBucket = "/" + volumeName + "/" + bucketName;
+ FSDataOutputStream[] streams = new FSDataOutputStream[numKeys];
+
+ try {
+ // Create multiple keys and hold them open
+ for (int i = 0; i < numKeys; i++) {
+ streams[i] = fs.create(new Path(keys[i]));
+ streams[i].write(1);
+ }
+
+ // Wait for DB flush
+ cluster.getOzoneManager().awaitDoubleBufferFlush();
+
+ // hsync last key
+ streams[numKeys - 1].hsync();
+ // Wait for flush
+ cluster.getOzoneManager().awaitDoubleBufferFlush();
+ final String[] args = new String[] {"om", "lof", "--service-id",
+ omServiceId, "--show-deleted", "--show-overwritten", "-p",
pathToBucket};
+
+ execute(ozoneAdminShell, args);
+ String cmdRes = getStdOut();
+
+ // Verify that key is hsync'ed
+ assertTrue(cmdRes.contains("\tYes\t\tNo\t\tNo"), "key should be hsync'ed
and not deleted, not overwritten");
+
+ execute(ozoneAdminShell, new String[] {"om", "lof", "--service-id",
+ omServiceId, "--show-overwritten", "-p", pathToBucket});
+ cmdRes = getStdOut();
+ // Verify that key is hsync'ed
+ assertTrue(cmdRes.contains("\tYes\t\tNo"), "key should be hsync'ed and
not overwritten");
+
+ // Verify json output
+ String[] args1 = new String[] {"om", "lof", "--service-id", omServiceId,
"--show-deleted", "--show-overwritten",
+ "--json", "-p", pathToBucket};
+ execute(ozoneAdminShell, args1);
+ cmdRes = getStdOut();
+
+ assertTrue(!cmdRes.contains(OzoneConsts.DELETED_HSYNC_KEY),
+ "key should not have deletedHsyncKey metadata");
+ assertTrue(!cmdRes.contains(OzoneConsts.OVERWRITTEN_HSYNC_KEY),
+ "key should not have overwrittenHsyncKey metadata");
+
+ // Suspend open key cleanup service so that key remains in openKeyTable
for verification
+ OpenKeyCleanupService openKeyCleanupService =
+ (OpenKeyCleanupService)
cluster.getOzoneManager().getKeyManager().getOpenKeyCleanupService();
+ openKeyCleanupService.suspend();
+ // overwrite last key
+ try (FSDataOutputStream os = fs.create(new Path(keys[numKeys - 1]))) {
+ os.write(2);
+ }
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ execute(ozoneAdminShell, args);
+ String cmdRes1 = getStdOut();
+ // When hsync file is overwritten, it should add
OVERWRITTEN_HSYNC_KEY metadata in hsync openKey
+ // And list open key should show as overwritten
+ return cmdRes1.contains("\tYes\t\tNo\t\tYes");
+ } catch (Throwable t) {
+ LOG.warn("Failed to list open key", t);
+ return false;
+ }
+ }, 1000, 10000);
+
+ // Now check json output
+ execute(ozoneAdminShell, args1);
+ cmdRes = getStdOut();
+ assertTrue(!cmdRes.contains(OzoneConsts.DELETED_HSYNC_KEY),
+ "key should not have deletedHsyncKey metadata");
+ assertTrue(cmdRes.contains(OzoneConsts.OVERWRITTEN_HSYNC_KEY),
+ "key should have overwrittenHsyncKey metadata");
+
+ // Verify result should not have overwritten hsync keys when
--show-overwritten is not in the command argument
+ String[] args2 = new String[] {"om", "lof", "--service-id", omServiceId,
"-p", pathToBucket};
+ execute(ozoneAdminShell, args2);
+ cmdRes = getStdOut();
+ // Verify that overwrittenHsyncKey is not in the result
+ assertTrue(!cmdRes.contains("\tYes\t\tYes"), "key should be hsync'ed and
not overwritten");
+
+ // Verify with json result
+ args2 = new String[] {"om", "lof", "--service-id", omServiceId,
"--json", "-p", pathToBucket};
+ execute(ozoneAdminShell, args2);
+ cmdRes = getStdOut();
+ // Verify that overwrittenHsyncKey is not in the result
+ assertTrue(!cmdRes.contains(OzoneConsts.OVERWRITTEN_HSYNC_KEY),
+ "key should not have overwrittenHsyncKey metadata");
+
+ } finally {
+ // Cleanup
+ IOUtils.closeQuietly(streams);
+ }
+ }
+
/**
* Return stdout as a String, then clears existing output.
*/
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/ListOpenFilesSubCommand.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/ListOpenFilesSubCommand.java
index 90b064fce6..723a4ec402 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/ListOpenFilesSubCommand.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/ListOpenFilesSubCommand.java
@@ -28,6 +28,7 @@ import
org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import picocli.CommandLine;
import java.io.IOException;
+import java.time.Instant;
import java.util.List;
import java.util.concurrent.Callable;
@@ -71,6 +72,11 @@ public class ListOpenFilesSubCommand implements
Callable<Void> {
description = "Whether to show deleted open keys")
private boolean showDeleted;
+ @CommandLine.Option(names = { "--show-overwritten" },
+ defaultValue = "false",
+ description = "Whether to show overwritten open keys")
+ private boolean showOverwritten;
+
// Conforms to ListOptions, but not all in ListOptions applies here thus
// not using that directly
@CommandLine.Option(
@@ -100,7 +106,7 @@ public class ListOpenFilesSubCommand implements
Callable<Void> {
public Void call() throws Exception {
if (StringUtils.isEmpty(omServiceId) && StringUtils.isEmpty(omHost)) {
- System.err.println("Error: Please specify -id or -host");
+ System.err.println("Error: Please specify --service-id or
--service-host");
return null;
}
@@ -113,6 +119,9 @@ public class ListOpenFilesSubCommand implements
Callable<Void> {
if (!showDeleted) {
res.getOpenKeys().removeIf(o ->
o.getKeyInfo().getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY));
}
+ if (!showOverwritten) {
+ res.getOpenKeys().removeIf(o ->
o.getKeyInfo().getMetadata().containsKey(OzoneConsts.OVERWRITTEN_HSYNC_KEY));
+ }
if (json) {
// Print detailed JSON
printOpenKeysListAsJson(res);
@@ -140,14 +149,16 @@ public class ListOpenFilesSubCommand implements
Callable<Void> {
if (startItem != null && !startItem.isEmpty()) {
msg += "\nafter continuation token:\n " + startItem;
}
- msg += showDeleted ? "\n\nClient ID\t\tCreation
time\tHsync'ed\tDeleted\t\tOpen File Path" :
- "\n\nClient ID\t\tCreation time\tHsync'ed\tOpen File Path";
+ msg += "\n\nClient ID\t\t\tCreation time\t\tHsync'ed\t";
+ msg += showDeleted ? "Deleted\t" : "";
+ msg += showOverwritten ? "Overwritten\t" : "";
+ msg += "Open File Path";
System.out.println(msg);
for (OpenKeySession e : openFileList) {
long clientId = e.getId();
OmKeyInfo omKeyInfo = e.getKeyInfo();
- String line = clientId + "\t" + omKeyInfo.getCreationTime() + "\t";
+ String line = clientId + "\t" +
Instant.ofEpochMilli(omKeyInfo.getCreationTime()) + "\t";
if (omKeyInfo.isHsync()) {
String hsyncClientIdStr =
@@ -168,8 +179,16 @@ public class ListOpenFilesSubCommand implements
Callable<Void> {
line += "No\t\t";
}
}
+ if (showOverwritten) {
+ if
(omKeyInfo.getMetadata().containsKey(OzoneConsts.OVERWRITTEN_HSYNC_KEY)) {
+ line += "Yes\t";
+ } else {
+ line += "No\t";
+ }
+ }
} else {
line += showDeleted ? "No\t\tNo\t\t" : "No\t\t";
+ line += showOverwritten ? "No\t" : "";
}
line += getFullPathFromKeyInfo(omKeyInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
