steveloughran commented on code in PR #6407:
URL: https://github.com/apache/hadoop/pull/6407#discussion_r1484732749
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java:
##########
@@ -27,19 +27,25 @@
/**
* <pre>
- * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the
{@code fs.s3a.glacier.read.restored.objects} configuration set in {@link
S3AFileSystem}
+ * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the
+ * {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link
S3AFileSystem}
* The config can have 3 values:
- * {@code READ_ALL}: Retrieval of Glacier files will fail with
InvalidObjectStateException: The operation is not valid for the object's
storage class.
- * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3
Objects which are tagged with Glacier storage classes and retrieve the others.
- * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored
status of the Glacier object will be checked, if restored the objects would be
read like normal S3 objects else they will be ignored as the objects would not
have been retrieved from the S3 Glacier.
+ * {@code READ_ALL}: Retrieval of Glacier files will fail with
InvalidObjectStateException:
+ * The operation is not valid for the object's storage class.
+ * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3
Objects which are
+ * tagged with Glacier storage classes and retrieve the others.
+ * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored
status of the Glacier
+ * object will be checked, if restored the objects would be read like normal
S3 objects
+ * else they will be ignored as the objects would not have been retrieved from
the S3 Glacier.
* </pre>
*/
public enum S3ObjectStorageClassFilter {
Review Comment:
I like this design you know: enum based mapping to closures.
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path dir = methodPath();
+ fs.mkdirs(dir);
+ Path path = new Path(dir, "file1");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS);
Review Comment:
add READ_RESTORED_GLACIER_OBJECTS to the varags list
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java:
##########
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
Review Comment:
nit: import ordering
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java:
##########
@@ -577,6 +583,11 @@ public void initialize(URI name, Configuration
originalConf)
s3aInternals = createS3AInternals();
+ s3ObjectStorageClassFilter =
Optional.of(conf.getTrimmed(READ_RESTORED_GLACIER_OBJECTS,
Review Comment:
what is the default today? we fail? we skip?
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
Review Comment:
no need to use this.
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
Review Comment:
ordering doesn't match the hadoop ones
```
java.*
javax.*
non-apache
org.apache
static imports
```
the way we've used search and replace to move off guava has tainted this a
bit, and don't reorder existing code as it makes cherrypicking awful, but for
new code, let's start with the ideal goal
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java:
##########
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
Review Comment:
can you put this in `org.apache.hadoop.fs.s3a.api'? I'd like to separate
things a bit more, and as StoreContext is used by plugins, it'd be appropriate
to move it in here
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path dir = methodPath();
+ fs.mkdirs(dir);
+ Path path = new Path(dir, "file1");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS);
+ return newConf;
+ }
+
+ @Test
+ public void testIgnoreGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreRestoringGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testRestoredGlacierObject() throws Throwable {
+ // Skipping this test for Deep Archive as expedited retrieval is not
supported
+ Assume.assumeTrue(type == Type.GLACIER);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ restoreGlacierObject(getFilePrefixForListObjects(), 2);
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testDefault() throws Throwable {
Review Comment:
the type info should go into the method name as well, so if skipped we know
what "default" means
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java:
##########
@@ -577,6 +583,11 @@ public void initialize(URI name, Configuration
originalConf)
s3aInternals = createS3AInternals();
+ s3ObjectStorageClassFilter =
Optional.of(conf.getTrimmed(READ_RESTORED_GLACIER_OBJECTS,
+ S3ObjectStorageClassFilter.READ_ALL.toString()))
Review Comment:
move this into Constants.java directly beneath
`READ_RESTORED_GLACIER_OBJECTS` into a string constant
`DEFAULT_READ_RESTORED_GLACIER_OBJECTS`.
Keeps options and defaults together for understanding what is configured
##########
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md:
##########
@@ -1309,6 +1309,20 @@ The switch to turn S3A auditing on or off.
</description>
</property>
+<!--
Review Comment:
how about adding this in its own little section
```
## <a name="glacier"> Glacier Object Support
```
* add a link to AWS glacier storage docs
* move the config up to text outside the xml
* note it is not available on all AWS S3 store types, or on third party
stores. (not on S3 express, is it)?
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path dir = methodPath();
+ fs.mkdirs(dir);
+ Path path = new Path(dir, "file1");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS);
+ return newConf;
+ }
+
+ @Test
+ public void testIgnoreGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreRestoringGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testRestoredGlacierObject() throws Throwable {
+ // Skipping this test for Deep Archive as expedited retrieval is not
supported
+ Assume.assumeTrue(type == Type.GLACIER);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ restoreGlacierObject(getFilePrefixForListObjects(), 2);
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testDefault() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_ALL.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+
+ private void restoreGlacierObject(String glacierObjectKey, int
expirationDays) throws Exception {
Review Comment:
can you add a javadoc. what happens if an object isn't there? timeout,
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path dir = methodPath();
+ fs.mkdirs(dir);
+ Path path = new Path(dir, "file1");
Review Comment:
call it "glaciated" so test failures are a bit more meaningful
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
