This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 36a80f5a0e6 HDDS-11421. `ozone dtutil get` to handle o3:// and ofs://
URLs (#11067)
36a80f5a0e6 is described below
commit 36a80f5a0e609892540a895b7fff74f3af077eea
Author: Eric C. Ho <[email protected]>
AuthorDate: Thu Aug 27 23:19:53 2026 +0800
HDDS-11421. `ozone dtutil get` to handle o3:// and ofs:// URLs (#11067)
Generated-by: Codex (GPT-5)
---
.../smoketest/security/ozone-secure-token.robot | 11 ++
...sDtFetcher.java => AbstractOzoneDtFetcher.java} | 41 ++----
.../org/apache/hadoop/fs/ozone/O3DtFetcher.java | 48 +++++++
.../org/apache/hadoop/fs/ozone/O3fsDtFetcher.java | 48 +------
.../org/apache/hadoop/fs/ozone/OfsDtFetcher.java | 31 +++++
.../apache/hadoop/fs/ozone/TestOzoneDtFetcher.java | 140 +++++++++++++++++++++
.../org.apache.hadoop.security.token.DtFetcher | 2 +
.../org.apache.hadoop.security.token.DtFetcher | 2 +
.../hadoop/fs/ozone/TestDtFetcherProviders.java | 41 ++++++
9 files changed, 289 insertions(+), 75 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
index a3fd5150bca..cc670262863 100644
--- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
@@ -24,6 +24,8 @@ Suite Setup Get Security Enabled From Config
*** Variables ***
${TOKEN_FILE} ${TEMP_DIR}/ozone.token
+${DTUTIL_OFS_TOKEN_FILE} ${TEMP_DIR}/dtutil-ofs.token
+${DTUTIL_O3_TOKEN_FILE} ${TEMP_DIR}/dtutil-o3.token
*** Keywords ***
Get and use Token in Secure Cluster
@@ -49,6 +51,13 @@ Print Valid Token File
${output} = Execute ozone sh token print -t
${TOKEN_FILE}
Should Not Be Empty ${output}
+Get Token With Dtutil
+ [Arguments] ${uri} ${token_file}
+ Remove File ${token_file}
+ Execute ozone dtutil get ${uri} ${token_file}
+ ${output} = Execute ozone dtutil print
${token_file}
+ Should Contain ${output} OzoneToken
+
Print Nonexistent Token File
${output} = Execute ozone sh token print -t
/asdf
Should Contain ${output} operation failed as token
file: /asdf
@@ -74,6 +83,8 @@ Cancel Token in Unsecure Cluster
Token Test in Secure Cluster
Get and use Token in Secure Cluster
Print Valid Token File
+ Get Token With Dtutil ofs://${OM_SERVICE_ID}/
${DTUTIL_OFS_TOKEN_FILE}
+ Get Token With Dtutil o3://${OM_SERVICE_ID}/
${DTUTIL_O3_TOKEN_FILE}
Renew Token in Secure Cluster
Cancel Token in Secure Cluster
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/AbstractOzoneDtFetcher.java
similarity index 62%
copy from
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
copy to
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/AbstractOzoneDtFetcher.java
index 57b4a2c7082..7980021ad96 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/AbstractOzoneDtFetcher.java
@@ -21,8 +21,6 @@
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.DtFetcher;
@@ -31,47 +29,34 @@
import org.slf4j.LoggerFactory;
/**
- * A DT fetcher for OzoneFileSystem.
- * It is only needed for the `hadoop dtutil` command.
+ * Base DT fetcher for Ozone URL schemes.
*/
-public class O3fsDtFetcher implements DtFetcher {
+abstract class AbstractOzoneDtFetcher implements DtFetcher {
private static final Logger LOG =
- LoggerFactory.getLogger(O3fsDtFetcher.class);
-
- private static final String SERVICE_NAME = OzoneConsts.OZONE_URI_SCHEME;
+ LoggerFactory.getLogger(AbstractOzoneDtFetcher.class);
private static final String FETCH_FAILED =
"Fetch ozone delegation token failed";
- /**
- * Returns the service name for O3fs, which is also a valid URL prefix.
- */
- @Override
- public Text getServiceName() {
- return new Text(SERVICE_NAME);
- }
-
@Override
public boolean isTokenRequired() {
return UserGroupInformation.isSecurityEnabled();
}
- /**
- * Returns Token object via FileSystem, null if bad argument.
- * @param conf - a Configuration object used with FileSystem.get()
- * @param creds - a Credentials object to which token(s) will be added
- * @param renewer - the renewer to send with the token request
- * @param url - the URL to which the request is sent
- * @return a Token, or null if fetch fails.
- */
@Override
public Token<?> addDelegationTokens(Configuration conf, Credentials creds,
String renewer, String url) throws Exception {
- if (!url.startsWith(getServiceName().toString())) {
- url = getServiceName().toString() + "://" + url;
+ String serviceName = getServiceName().toString();
+ if (!url.startsWith(serviceName + "://")) {
+ url = serviceName + "://" + url;
}
- LOG.debug("addDelegationTokens from {} renewer {}.", url, renewer);
- FileSystem fs = FileSystem.get(URI.create(url), conf);
+ return addDelegationTokens(conf, creds, renewer, URI.create(url));
+ }
+
+ protected Token<?> addDelegationTokens(Configuration conf, Credentials creds,
+ String renewer, URI uri) throws IOException {
+ LOG.debug("addDelegationTokens from {} renewer {}.", uri, renewer);
+ FileSystem fs = FileSystem.get(uri, conf);
Token<?> token = fs.getDelegationToken(renewer);
if (token == null) {
LOG.error(FETCH_FAILED);
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3DtFetcher.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3DtFetcher.java
new file mode 100644
index 00000000000..0fed84114fc
--- /dev/null
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3DtFetcher.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.net.URI;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * A DT fetcher for Ozone RPC URLs.
+ */
+public class O3DtFetcher extends AbstractOzoneDtFetcher {
+ @Override
+ public Text getServiceName() {
+ return new Text(OzoneConsts.OZONE_RPC_SCHEME);
+ }
+
+ @Override
+ protected Token<?> addDelegationTokens(Configuration conf, Credentials creds,
+ String renewer, URI uri) throws IOException {
+ if (uri.getAuthority() == null) {
+ throw new IllegalArgumentException(
+ "OM authority is required in Ozone RPC URL: " + uri);
+ }
+ URI ofsUri = URI.create(OzoneConsts.OZONE_OFS_URI_SCHEME + "://"
+ + uri.getRawAuthority() + "/");
+ return super.addDelegationTokens(conf, creds, renewer, ofsUri);
+ }
+}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
index 57b4a2c7082..48c9e01a0d4 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
@@ -17,32 +17,16 @@
package org.apache.hadoop.fs.ozone;
-import java.io.IOException;
-import java.net.URI;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.DtFetcher;
-import org.apache.hadoop.security.token.Token;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A DT fetcher for OzoneFileSystem.
* It is only needed for the `hadoop dtutil` command.
*/
-public class O3fsDtFetcher implements DtFetcher {
- private static final Logger LOG =
- LoggerFactory.getLogger(O3fsDtFetcher.class);
-
+public class O3fsDtFetcher extends AbstractOzoneDtFetcher {
private static final String SERVICE_NAME = OzoneConsts.OZONE_URI_SCHEME;
- private static final String FETCH_FAILED =
- "Fetch ozone delegation token failed";
-
/**
* Returns the service name for O3fs, which is also a valid URL prefix.
*/
@@ -50,34 +34,4 @@ public class O3fsDtFetcher implements DtFetcher {
public Text getServiceName() {
return new Text(SERVICE_NAME);
}
-
- @Override
- public boolean isTokenRequired() {
- return UserGroupInformation.isSecurityEnabled();
- }
-
- /**
- * Returns Token object via FileSystem, null if bad argument.
- * @param conf - a Configuration object used with FileSystem.get()
- * @param creds - a Credentials object to which token(s) will be added
- * @param renewer - the renewer to send with the token request
- * @param url - the URL to which the request is sent
- * @return a Token, or null if fetch fails.
- */
- @Override
- public Token<?> addDelegationTokens(Configuration conf, Credentials creds,
- String renewer, String url) throws Exception {
- if (!url.startsWith(getServiceName().toString())) {
- url = getServiceName().toString() + "://" + url;
- }
- LOG.debug("addDelegationTokens from {} renewer {}.", url, renewer);
- FileSystem fs = FileSystem.get(URI.create(url), conf);
- Token<?> token = fs.getDelegationToken(renewer);
- if (token == null) {
- LOG.error(FETCH_FAILED);
- throw new IOException(FETCH_FAILED);
- }
- creds.addToken(token.getService(), token);
- return token;
- }
}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OfsDtFetcher.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OfsDtFetcher.java
new file mode 100644
index 00000000000..da75f8ba328
--- /dev/null
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OfsDtFetcher.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * A DT fetcher for RootedOzoneFileSystem.
+ */
+public class OfsDtFetcher extends AbstractOzoneDtFetcher {
+ @Override
+ public Text getServiceName() {
+ return new Text(OzoneConsts.OZONE_OFS_URI_SCHEME);
+ }
+}
diff --git
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDtFetcher.java
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDtFetcher.java
new file mode 100644
index 00000000000..1fd4550d011
--- /dev/null
+++
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDtFetcher.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.DtFetcher;
+import org.apache.hadoop.security.token.Token;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.MockedStatic;
+
+/**
+ * Tests Ozone delegation token fetchers.
+ */
+public class TestOzoneDtFetcher {
+ private static final String RENEWER = "renewer";
+
+ @ParameterizedTest
+ @MethodSource("fetchers")
+ public void fetchesTokenFromExpectedFileSystem(DtFetcher fetcher,
+ String url, URI expectedUri) throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ Credentials creds = new Credentials();
+ FileSystem fs = mock(FileSystem.class);
+ Token<?> token = new Token<>();
+ Text service = new Text("om-service");
+ token.setService(service);
+ doReturn(token).when(fs).getDelegationToken(RENEWER);
+
+ try (MockedStatic<FileSystem> fileSystems = mockStatic(FileSystem.class)) {
+ fileSystems.when(() -> FileSystem.get(expectedUri, conf)).thenReturn(fs);
+
+ assertSame(token,
+ fetcher.addDelegationTokens(conf, creds, RENEWER, url));
+ assertSame(token, creds.getToken(service));
+ fileSystems.verify(() -> FileSystem.get(expectedUri, conf));
+ }
+ }
+
+ private static Stream<Arguments> fetchers() {
+ return Stream.of(
+ Arguments.of(new O3fsDtFetcher(),
+ "o3fs://bucket.volume.om/key",
+ URI.create("o3fs://bucket.volume.om/key")),
+ Arguments.of(new OfsDtFetcher(),
+ "ofs://om/volume/bucket/key",
+ URI.create("ofs://om/volume/bucket/key")),
+ Arguments.of(new O3DtFetcher(),
+ "o3://om-service/volume/bucket/key",
+ URI.create("ofs://om-service/")),
+ Arguments.of(new O3DtFetcher(),
+ "om-service/volume/bucket/key",
+ URI.create("ofs://om-service/")),
+ Arguments.of(new O3DtFetcher(),
+ "o3://om:9862/volume/bucket/key?query#fragment",
+ URI.create("ofs://om:9862/")));
+ }
+
+ @Test
+ public void checksFullServiceNamePrefix() throws Exception {
+ AbstractOzoneDtFetcher fetcher = new AbstractOzoneDtFetcher() {
+ @Override
+ public Text getServiceName() {
+ return new Text("o3");
+ }
+
+ @Override
+ protected Token<?> addDelegationTokens(Configuration conf,
+ Credentials creds, String renewer, URI uri) {
+ assertEquals(URI.create("o3://o3fs://bucket.volume.om/key"), uri);
+ return null;
+ }
+ };
+
+ fetcher.addDelegationTokens(new OzoneConfiguration(), new Credentials(),
+ RENEWER, "o3fs://bucket.volume.om/key");
+ }
+
+ @Test
+ public void hasExpectedServiceNames() {
+ assertEquals(new Text("o3fs"), new O3fsDtFetcher().getServiceName());
+ assertEquals(new Text("ofs"), new OfsDtFetcher().getServiceName());
+ assertEquals(new Text("o3"), new O3DtFetcher().getServiceName());
+ }
+
+ @Test
+ public void rejectsO3UrlWithoutAuthority() {
+ O3DtFetcher fetcher = new O3DtFetcher();
+ assertThrows(IllegalArgumentException.class,
+ () -> fetcher.addDelegationTokens(new OzoneConfiguration(),
+ new Credentials(), RENEWER, "o3:///"));
+ }
+
+ @Test
+ public void rejectsNullToken() throws Exception {
+ Configuration conf = new OzoneConfiguration();
+ Credentials creds = new Credentials();
+ FileSystem fs = mock(FileSystem.class);
+ URI uri = URI.create("ofs://om/");
+ doReturn(null).when(fs).getDelegationToken(RENEWER);
+
+ try (MockedStatic<FileSystem> fileSystems = mockStatic(FileSystem.class)) {
+ fileSystems.when(() -> FileSystem.get(uri, conf)).thenReturn(fs);
+ assertThrows(IOException.class,
+ () -> new OfsDtFetcher().addDelegationTokens(
+ conf, creds, RENEWER, uri.toString()));
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
b/hadoop-ozone/ozonefs-hadoop3/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
index 6e867319c17..8ba2d0d63bf 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -17,3 +17,5 @@
#
org.apache.hadoop.fs.ozone.O3fsDtFetcher
+org.apache.hadoop.fs.ozone.OfsDtFetcher
+org.apache.hadoop.fs.ozone.O3DtFetcher
diff --git
a/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
b/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
index 6e867319c17..8ba2d0d63bf 100644
---
a/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
+++
b/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -17,3 +17,5 @@
#
org.apache.hadoop.fs.ozone.O3fsDtFetcher
+org.apache.hadoop.fs.ozone.OfsDtFetcher
+org.apache.hadoop.fs.ozone.O3DtFetcher
diff --git
a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestDtFetcherProviders.java
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestDtFetcherProviders.java
new file mode 100644
index 00000000000..38a3ddd722e
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestDtFetcherProviders.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.HashSet;
+import java.util.ServiceLoader;
+import java.util.Set;
+import org.apache.hadoop.security.token.DtFetcher;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests Ozone delegation token fetcher registrations.
+ */
+public class TestDtFetcherProviders {
+ @Test
+ public void loadsOzoneDtFetchers() {
+ Set<String> serviceNames = new HashSet<>();
+ for (DtFetcher fetcher : ServiceLoader.load(DtFetcher.class)) {
+ serviceNames.add(fetcher.getServiceName().toString());
+ }
+
+ assertThat(serviceNames).contains("o3", "ofs", "o3fs");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
