This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7b9d30eec7 Fix GCS Folder deletion for both HNS Enabled and HNS
Disabled buckets (#7213)
7b9d30eec7 is described below
commit 7b9d30eec720bcddfe69047a9020d808219d621e
Author: Josh Downing <[email protected]>
AuthorDate: Tue Jun 2 09:49:47 2026 -0400
Fix GCS Folder deletion for both HNS Enabled and HNS Disabled buckets
(#7213)
* folders should now get deleted in a hierarchical namespace enabled bucket
* Fix delete folders on gcs for hns enabled and hns not enabled
* Fix delete folders on gcs for hns enabled and hns not enabled
* Fix delete folders on gcs for hns enabled and hns not enabled
---
plugins/tech/google/pom.xml | 18 ++++++
.../apache/hop/vfs/gs/GoogleStorageFileObject.java | 65 ++++++++++++++++------
.../apache/hop/vfs/gs/GoogleStorageFileSystem.java | 20 +++++++
3 files changed, 87 insertions(+), 16 deletions(-)
diff --git a/plugins/tech/google/pom.xml b/plugins/tech/google/pom.xml
index e48c754494..3236dbcdc7 100755
--- a/plugins/tech/google/pom.xml
+++ b/plugins/tech/google/pom.xml
@@ -157,6 +157,24 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage-control</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.conscrypt</groupId>
+ <artifactId>conscrypt-openjdk-uber</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileObject.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileObject.java
index 432807cc81..f487dd96c7 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileObject.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileObject.java
@@ -19,6 +19,7 @@
package org.apache.hop.vfs.gs;
import com.google.api.gax.paging.Page;
+import com.google.api.gax.rpc.NotFoundException;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
@@ -26,6 +27,9 @@ import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
+import com.google.storage.control.v2.DeleteFolderRequest;
+import com.google.storage.control.v2.FolderName;
+import com.google.storage.control.v2.StorageControlClient;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -200,6 +204,9 @@ public class GoogleStorageFileObject extends
AbstractFileObject<GoogleStorageFil
}
Map<String, GoogleStorageListCache.ChildInfo> cacheEntries = new
LinkedHashMap<>();
for (Blob b : page.iterateAll()) {
+ if
(stripTrailingSlash(b.getName()).equals(stripTrailingSlash(bucketPath))) {
+ continue;
+ }
results.add(lastPathElement(stripTrailingSlash(b.getName())));
FileType childType = b.isDirectory() ? FileType.FOLDER : FileType.FILE;
long childSize = b.getSize() != null ? b.getSize() : 0L;
@@ -233,18 +240,55 @@ public class GoogleStorageFileObject extends
AbstractFileObject<GoogleStorageFil
protected void doDelete() throws Exception {
if (hasObject()) {
blob.delete();
+ if (isHnsEnabled()) {
+ handleHnsDirectoryDeletion(stripLeadingSlash(bucketPath));
+ }
} else if (bucketName != null
&& !bucketName.isEmpty()
&& bucketPath != null
&& !bucketPath.isEmpty()) {
Storage storage = getAbstractFileSystem().setupStorage();
- storage.delete(BlobId.of(bucketName, stripLeadingSlash(bucketPath)));
+ String path = stripLeadingSlash(bucketPath);
+ Blob found = findBlob(storage, path);
+ if (found != null) {
+ found.delete();
+ }
+ if (isHnsEnabled()) {
+ handleHnsDirectoryDeletion(path);
+ }
} else {
throw new IOException("Cannot delete: missing bucket/object path for '"
+ this + "'");
}
getAbstractFileSystem().invalidateListCacheForParentOf(bucketName,
bucketPath);
}
+ private Blob findBlob(Storage storage, String path) {
+ Blob found = storage.get(BlobId.of(bucketName, path));
+ if (found == null) {
+ found = storage.get(BlobId.of(bucketName, path + "/"));
+ }
+ return found;
+ }
+
+ private void handleHnsDirectoryDeletion(String path) throws IOException {
+ String folderName = FolderName.format("_", bucketName,
stripTrailingSlash(path));
+ try {
+ StorageControlClient controlClient =
getAbstractFileSystem().getStorageControlClient();
+ DeleteFolderRequest folderRequest =
+ DeleteFolderRequest.newBuilder().setName(folderName).build();
+ controlClient.deleteFolder(folderRequest);
+ } catch (NotFoundException e) {
+ // Not an HNS folder, skip.
+ }
+ }
+
+ private boolean isHnsEnabled() {
+ if (bucket == null || bucket.getHierarchicalNamespace() == null) {
+ return false;
+ }
+ return bucket.getHierarchicalNamespace().getEnabled();
+ }
+
@Override
protected void doDetach() throws Exception {
bucket = null;
@@ -316,11 +360,7 @@ public class GoogleStorageFileObject extends
AbstractFileObject<GoogleStorageFil
if (!hasBucket()) {
Page<Bucket> page = storage.list();
for (Bucket b : page.iterateAll()) {
- results.add(
- new GoogleStorageFileObject(
- scheme,
- new GoogleStorageFileName(scheme, b.getName(),
FileType.FOLDER),
- getAbstractFileSystem()));
+ results.add(getAbstractFileSystem().resolveFile("/" + b.getName()));
}
} else {
String prefix;
@@ -339,7 +379,7 @@ public class GoogleStorageFileObject extends
AbstractFileObject<GoogleStorageFil
}
Map<String, GoogleStorageListCache.ChildInfo> cacheEntries = new
LinkedHashMap<>();
for (Blob b : page.iterateAll()) {
- if (this.blob != null && b.getName().equals(this.blob.getName())) {
+ if
(stripTrailingSlash(b.getName()).equals(stripTrailingSlash(bucketPath))) {
continue;
}
FileType childType = b.isDirectory() ? FileType.FOLDER : FileType.FILE;
@@ -351,15 +391,8 @@ public class GoogleStorageFileObject extends
AbstractFileObject<GoogleStorageFil
cacheEntries.put(
b.getName(), new GoogleStorageListCache.ChildInfo(childType,
childSize, childLastMod));
results.add(
- new GoogleStorageFileObject(
- scheme,
- new GoogleStorageFileName(
- scheme,
- getName().getPath() + "/" +
lastPathElement(stripTrailingSlash(b.getName())),
- childType),
- getAbstractFileSystem(),
- this.bucket != null ? bucket : storage.get(bucketName),
- b));
+ getAbstractFileSystem()
+ .resolveFile("/" + bucketName + "/" +
stripTrailingSlash(b.getName())));
}
if (!cacheEntries.isEmpty()) {
getAbstractFileSystem().putListCache(bucketName, prefix, cacheEntries);
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
index 61b2d41b4e..72d2d7103a 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageFileSystem.java
@@ -18,10 +18,14 @@
package org.apache.hop.vfs.gs;
+import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
+import com.google.storage.control.v2.StorageControlClient;
+import com.google.storage.control.v2.StorageControlSettings;
+import java.io.IOException;
import java.util.Collection;
import org.apache.commons.vfs2.Capability;
import org.apache.commons.vfs2.FileName;
@@ -38,6 +42,7 @@ import org.threeten.bp.Duration;
public class GoogleStorageFileSystem extends AbstractFileSystem {
Storage storage = null;
+ StorageControlClient storageControlClient = null;
FileSystemOptions fileSystemOptions;
private GoogleStorageListCache listCache;
@@ -150,4 +155,19 @@ public class GoogleStorageFileSystem extends
AbstractFileSystem {
return "";
}
}
+
+ StorageControlClient getStorageControlClient() throws IOException {
+ if (storageControlClient != null) {
+ return storageControlClient;
+ }
+ StorageControlSettings settings =
+ StorageControlSettings.newBuilder()
+ .setCredentialsProvider(
+ FixedCredentialsProvider.create(
+ GoogleStorageFileSystemConfigBuilder.getInstance()
+ .getGoogleCredentials(fileSystemOptions)))
+ .build();
+ storageControlClient = StorageControlClient.create(settings);
+ return storageControlClient;
+ }
}
