peterxcli commented on code in PR #8307: URL: https://github.com/apache/ozone/pull/8307#discussion_r2051750060
########## hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/rest/S3V4Signer.java: ########## @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.rest; + +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** + * Utility for generating AWS S3 Signature V4 headers for HTTP requests. + * Only supports unsigned payloads (useful for GET/HEAD). + */ +public final class S3V4Signer { + private S3V4Signer() { + throw new AssertionError("Utility class"); + } + + /** + * Signs the given HttpURLConnection with AWS S3 Signature V4 headers. + * + * @param conn The HttpURLConnection to sign + * @param accessKey AWS access key + * @param secretKey AWS secret key + * @param region AWS region (e.g., "us-east-1") + * @param service AWS service (e.g., "s3") + * @param bucket S3 bucket name + * @param queryString The query string (e.g., "max-keys=-1") + */ + public static void signRequest(HttpURLConnection conn, String accessKey, String secretKey, String region, Review Comment: Do you have any reference for this signing process? ########## hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java: ########## @@ -292,6 +294,14 @@ public Response get( return Response.ok(response).build(); } + private int validateMaxKeys(int maxKeys) throws OS3Exception { + if (maxKeys <= 0) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0"); Review Comment: ```suggestion throw newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0"); ``` ########## hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/rest/TestS3RestNonSdkCases.java: ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.rest; + +import static org.apache.hadoop.ozone.OzoneConsts.LOCALHOST; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.HttpURLConnection; +import java.net.URL; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.s3.S3GatewayService; +import org.apache.ozone.test.OzoneTestBase; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for S3 REST edge cases that cannot be triggered by AWS SDK clients. + * For example: negative/zero max-keys, or other invalid parameters that SDK would block client-side. + */ +public class TestS3RestNonSdkCases extends OzoneTestBase { + + private static MiniOzoneCluster cluster = null; + private static S3GatewayService s3g = null; + private static final String ACCESS_KEY = "testuser"; + private static final String SECRET_KEY = "testpass"; + + @BeforeAll + public static void startCluster() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); + s3g = new S3GatewayService(); + + cluster = MiniOzoneCluster.newBuilder(conf) + .addService(s3g) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + cluster.newClient().getObjectStore().createS3Bucket(getTestBucketName()); + } + + @AfterAll + public static void shutdownCluster() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testListObjectsWithNegativeMaxKeys() throws Exception { + final String bucketName = getTestBucketName(); + String s3Endpoint = getS3EndpointURL(); Review Comment: This can move to @BeforeAll ########## hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/rest/TestS3RestNonSdkCases.java: ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.rest; + +import static org.apache.hadoop.ozone.OzoneConsts.LOCALHOST; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.HttpURLConnection; +import java.net.URL; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.s3.S3GatewayService; +import org.apache.ozone.test.OzoneTestBase; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for S3 REST edge cases that cannot be triggered by AWS SDK clients. + * For example: negative/zero max-keys, or other invalid parameters that SDK would block client-side. + */ +public class TestS3RestNonSdkCases extends OzoneTestBase { + + private static MiniOzoneCluster cluster = null; + private static S3GatewayService s3g = null; + private static final String ACCESS_KEY = "testuser"; + private static final String SECRET_KEY = "testpass"; + + @BeforeAll + public static void startCluster() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); + s3g = new S3GatewayService(); + + cluster = MiniOzoneCluster.newBuilder(conf) + .addService(s3g) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + cluster.newClient().getObjectStore().createS3Bucket(getTestBucketName()); + } + + @AfterAll + public static void shutdownCluster() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testListObjectsWithNegativeMaxKeys() throws Exception { + final String bucketName = getTestBucketName(); + String s3Endpoint = getS3EndpointURL(); + String queryString = "max-keys=-1"; + String url = s3Endpoint + "/" + bucketName + "/?" + queryString; Review Comment: Could you build this with url builder? ########## hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java: ########## @@ -292,6 +294,14 @@ public Response get( return Response.ok(response).build(); } + private int validateMaxKeys(int maxKeys) throws OS3Exception { + if (maxKeys <= 0) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0"); + } + + return Math.min(maxKeys, 1000); Review Comment: Same behaviour as AWS S3 does? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
