This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a998107a1f5 [Java] Add warning to gcpTempLocation when its bucket has
soft delete enabled (#31358)
a998107a1f5 is described below
commit a998107a1f5c3050821eef6a5ad5843d8adb8aec
Author: Shunping Huang <shunp...@google.com>
AuthorDate: Tue May 21 16:27:48 2024 -0400
[Java] Add warning to gcpTempLocation when its bucket has soft delete
enabled (#31358)
* Add warning to gcpTempLocation when its bucket has soft delete policy
enabled.
* Fix an issue reported by spotbugs.
---
.../sdk/extensions/gcp/options/GcpOptions.java | 34 +++++++++++++++++++++
.../sdk/extensions/gcp/options/GcpOptionsTest.java | 35 ++++++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git
a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 2686a53cbcd..c370606de37 100644
---
a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++
b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -42,6 +42,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory;
@@ -49,6 +50,7 @@ import
org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory;
import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer;
import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
import org.apache.beam.sdk.extensions.gcp.util.BackOffAdapter;
+import org.apache.beam.sdk.extensions.gcp.util.GcsUtil;
import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer;
import org.apache.beam.sdk.extensions.gcp.util.Transport;
import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath;
@@ -390,9 +392,41 @@ public interface GcpOptions extends GoogleApiDebugOptions,
PipelineOptions {
e);
}
}
+
+ if (isSoftDeletePolicyEnabled(options, tempLocation)) {
+ LOG.warn(
+ String.format(
+ "The bucket of gcpTempLocation %s has soft delete policy
enabled."
+ + " Dataflow jobs use Cloud Storage to store temporary
files during pipeline"
+ + " execution. To avoid being billed for unnecessary
storage costs, turn off the soft"
+ + " delete feature on buckets that your Dataflow jobs use
for temporary storage."
+ + " For more information, see"
+ + "
https://cloud.google.com/storage/docs/use-soft-delete#remove-soft-delete-policy.";,
+ tempLocation));
+ }
+
return tempLocation;
}
+ @VisibleForTesting
+ static boolean isSoftDeletePolicyEnabled(PipelineOptions options, String
tempLocation) {
+ GcsOptions gcsOptions = options.as(GcsOptions.class);
+ GcsUtil gcsUtil = gcsOptions.getGcsUtil();
+ try {
+ SoftDeletePolicy policy =
+
Objects.requireNonNull(gcsUtil.getBucket(GcsPath.fromUri(tempLocation)))
+ .getSoftDeletePolicy();
+ if (policy != null && policy.getRetentionDurationSeconds() > 0) {
+ return true;
+ }
+ } catch (Exception e) {
+ LOG.warn(
+ String.format(
+ "Failed to access bucket for gcpTempLocation: %s.%nCaused by
%s", tempLocation, e));
+ }
+ return false;
+ }
+
@VisibleForTesting
static ImmutableList<String> getDefaultBucketNameStubs(
PipelineOptions options, CloudResourceManager crmClient, String
bucketNamePrefix) {
diff --git
a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
index a182f0ab82c..3b6ed81bde8 100644
---
a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
+++
b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
@@ -19,7 +19,9 @@ package org.apache.beam.sdk.extensions.gcp.options;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Matchers.any;
@@ -35,6 +37,7 @@ import
com.google.api.services.cloudresourcemanager.CloudResourceManager.Project
import
com.google.api.services.cloudresourcemanager.CloudResourceManager.Projects.Get;
import com.google.api.services.cloudresourcemanager.model.Project;
import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Bucket.SoftDeletePolicy;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -244,6 +247,38 @@ public class GcpOptionsTest {
Long.valueOf(0L));
}
+ @Test
+ public void testTempLocationWithSoftDeletePolicy() throws IOException {
+ Bucket bucket = new Bucket();
+ bucket.setSoftDeletePolicy(new
SoftDeletePolicy().setRetentionDurationSeconds(1L));
+ when(mockGcsUtil.getBucket(any(GcsPath.class))).thenReturn(bucket);
+
+ String tempLocation = "gs://bucket_with_soft_delete";
+ options.setTempLocation(tempLocation);
+
options.as(GcsOptions.class).setPathValidatorClass(NoopPathValidator.class);
+
+ GcpOptions gcpOptions = options.as(GcpOptions.class);
+ assertEquals(tempLocation, gcpOptions.getGcpTempLocation());
+
+ assertTrue(GcpTempLocationFactory.isSoftDeletePolicyEnabled(options,
tempLocation));
+ }
+
+ @Test
+ public void testTempLocationWithoutSoftDeletePolicy() throws IOException {
+ Bucket bucket = new Bucket();
+ bucket.setSoftDeletePolicy(new
SoftDeletePolicy().setRetentionDurationSeconds(0L));
+ when(mockGcsUtil.getBucket(any(GcsPath.class))).thenReturn(bucket);
+
+ String tempLocation = "gs://bucket_without_soft_delete";
+ options.setTempLocation(tempLocation);
+
options.as(GcsOptions.class).setPathValidatorClass(NoopPathValidator.class);
+
+ GcpOptions gcpOptions = options.as(GcpOptions.class);
+ assertEquals(tempLocation, gcpOptions.getGcpTempLocation());
+
+ assertFalse(GcpTempLocationFactory.isSoftDeletePolicyEnabled(options,
tempLocation));
+ }
+
@Test
public void testCreateBucketProjectLookupFails() throws Exception {
doThrow(new IOException("badness")).when(mockGet).execute();
