Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Wail Alkowaileet :
Wail Alkowaileet has submitted this change. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 )
Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
..
[ASTERIXDB-3390][STO]: Update storage scheme for GCS
Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Savyasach Reddy
Reviewed-by: Wail Alkowaileet
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
5 files changed, 24 insertions(+), 10 deletions(-)
Approvals:
Wail Alkowaileet: Looks good to me, approved
Savyasach Reddy: Looks good to me, but someone else must approve
Jenkins: Verified; Verified
Anon. E. Moose #1000171:
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index d0ebd24..ccb7f9d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -64,7 +64,7 @@
cloud.deployment=true
storage.buffercache.pagesize=32KB
storage.partitioning=static
-cloud.storage.scheme=gcs
+cloud.storage.scheme=gs
cloud.storage.bucket=cloud-storage-container
cloud.storage.region=us-west2
cloud.storage.endpoint=http://127.0.0.1:4443
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 8cf5ac5..9df26f2 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -239,14 +239,17 @@
public final long doSyncWrite(IFileHandle fHandle, long offset,
ByteBuffer[] dataArray)
throws HyracksDataException {
long writtenBytes = localIoManager.doSyncWrite(fHandle, offset,
dataArray);
+dataArray[0].flip();
+dataArray[1].flip();
cloudWrite(fHandle, dataArray);
return writtenBytes;
}
@Override
-public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer
dataArray) throws HyracksDataException {
-int writtenBytes = localIoManager.doSyncWrite(fHandle, offset,
dataArray);
-cloudWrite(fHandle, dataArray);
+public final int doSyncWrite(IFileHandle fHandle, long offset, ByteBuffer
data) throws HyracksDataException {
+int writtenBytes = localIoManager.doSyncWrite(fHandle, offset, data);
+data.flip();
+cloudWrite(fHandle, data);
return writtenBytes;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 35ab467..d5f508d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -27,7 +27,7 @@
public class CloudClientProvider {
private static final String S3 = "s3";
-private static final String GCS = "gcs";
+private static final String GCS = "gs";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 9eb3fe0..9bcd386 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -204,7 +204,7 @@
public boolean exists(String bucket, String path) {
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path,
Storage.BlobGetOption.fields(Storage.BlobField.values()));
-return blob != null;
+return blob != null && blob.exists();
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index cccd9ec..a6dade5 100644
---
a
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Wail Alkowaileet : Attention is currently required from: Savyasach Reddy. Wail Alkowaileet has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Code-Review+2 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Savyasach Reddy Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 15:14:57 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy : Attention is currently required from: Wail Alkowaileet. Savyasach Reddy has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Code-Review+1 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Savyasach Reddy Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Wail Alkowaileet Gerrit-Comment-Date: Fri, 10 May 2024 12:43:07 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Contrib+1 Analytics Compatibility Tests Successful https://cbjenkins.page.link/Lriv3vEpUnhdJAXK8 : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 12:19:20 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Analytics Compatibility Tests Failed https://cbjenkins.page.link/N7fzUuuCqGbBS3YR7 : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 09:43:13 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 9: Contrib-2 Analytics Compatibility Tests Failed https://cbjenkins.page.link/QMas2JKH5D2Yp7fr5 : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 9 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 06:38:13 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Contrib-2 Analytics Compatibility Tests Failed https://cbjenkins.page.link/kpQNhGAUjetDLD487 : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 06:37:10 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/528/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 06:30:42 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Integration-Tests-1 Integration Tests Failed https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/525/ : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 05:04:09 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 10: Analytics Compatibility Compilation Successful https://cbjenkins.page.link/6HSoBpqLD1yef1q97 : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 10 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Fri, 10 May 2024 04:31:07 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 9: Contrib-2 Analytics Compatibility Tests Failed https://cbjenkins.page.link/po47p2hW6mXBLVE2A : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 9 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 19:08:45 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 1: Contrib-2 Analytics Compatibility Tests Failed https://cbjenkins.page.link/rZ9mixphrGuWf3qr9 : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 1 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 18:59:23 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 9: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/512/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 9 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 17:22:18 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
Attention is currently required from: Savyasach Reddy. Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 9: Analytics Compatibility Compilation Successful https://cbjenkins.page.link/x7ppXiuoY8wrieWf9 : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 9 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 16:50:46 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy : Attention is currently required from: Savyasach Reddy. Hello Wail Alkowaileet, Jenkins, Anon. E. Moose #1000171, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 to look at the new patch set (#9). Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. [ASTERIXDB-3390][STO]: Update storage scheme for GCS Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java 5 files changed, 19 insertions(+), 10 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/72/18272/9 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 9 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-MessageType: newpatchset
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy : Attention is currently required from: Savyasach Reddy. Hello Wail Alkowaileet, Jenkins, Anon. E. Moose #1000171, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 to look at the new patch set (#8). Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. [ASTERIXDB-3390][STO]: Update storage scheme for GCS Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java 5 files changed, 21 insertions(+), 10 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/72/18272/8 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 8 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-MessageType: newpatchset
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 7: Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/508/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 7 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 09:30:14 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy : Attention is currently required from: Savyasach Reddy. Hello Wail Alkowaileet, Jenkins, Anon. E. Moose #1000171, I'd like you to reexamine a change. Please visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 to look at the new patch set (#7). Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. [ASTERIXDB-3390][STO]: Update storage scheme for GCS Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java 5 files changed, 21 insertions(+), 10 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/72/18272/7 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 7 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-MessageType: newpatchset
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 6: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/507/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 6 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 06:43:50 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Wail Alkowaileet : Attention is currently required from: Savyasach Reddy. Wail Alkowaileet has uploaded a new patch set (#6) to the change originally created by Savyasach Reddy. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. [ASTERIXDB-3390][STO]: Update storage scheme for GCS Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java 5 files changed, 21 insertions(+), 10 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/72/18272/6 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 6 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-MessageType: newpatchset
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 5: Integration Tests Failed https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/505/ : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 5 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 04:19:09 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Wail Alkowaileet : Attention is currently required from: Savyasach Reddy. Wail Alkowaileet has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 5: Code-Review+2 -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 5 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Thu, 09 May 2024 00:02:56 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Jenkins : Attention is currently required from: Wail Alkowaileet, Savyasach Reddy. Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 5: Integration-Tests-1 Integration Tests Failed https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-trigger/501/ : UNSTABLE -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 5 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Wail Alkowaileet Gerrit-Attention: Savyasach Reddy Gerrit-Comment-Date: Wed, 08 May 2024 20:51:08 + Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy : Attention is currently required from: Wail Alkowaileet. Savyasach Reddy has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 ) Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS .. Patch Set 5: This change is ready for review. -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 5 Gerrit-Owner: Savyasach Reddy Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Wail Alkowaileet Gerrit-Attention: Wail Alkowaileet Gerrit-Comment-Date: Wed, 08 May 2024 20:33:04 + Gerrit-HasComments: No Gerrit-Has-Labels: No Gerrit-MessageType: comment
Change in asterixdb[master]: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
>From Savyasach Reddy :
Savyasach Reddy has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 )
Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS
..
[ASTERIXDB-3390][STO]: Update storage scheme for GCS
Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
4 files changed, 23 insertions(+), 19 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/72/18272/1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index d0ebd24..ccb7f9d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -64,7 +64,7 @@
cloud.deployment=true
storage.buffercache.pagesize=32KB
storage.partitioning=static
-cloud.storage.scheme=gcs
+cloud.storage.scheme=gs
cloud.storage.bucket=cloud-storage-container
cloud.storage.region=us-west2
cloud.storage.endpoint=http://127.0.0.1:4443
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index 35ab467..d5f508d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -27,7 +27,7 @@
public class CloudClientProvider {
private static final String S3 = "s3";
-private static final String GCS = "gcs";
+private static final String GCS = "gs";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index c725ca5..0bee4e9 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -32,7 +32,9 @@
import java.util.List;
import java.util.Set;
+import org.apache.asterix.cloud.CloudResettableInputStream;
import org.apache.asterix.cloud.IWriteBufferProvider;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.clients.ICloudWriter;
import org.apache.asterix.cloud.clients.IParallelDownloader;
@@ -87,7 +89,8 @@
@Override
public ICloudWriter createdWriter(String bucket, String path,
IWriteBufferProvider bufferProvider) {
-return new GCSWriter(bucket, path, gcsClient, profiler);
+ICloudBufferedWriter bufferedWriter = new GCSWriter(bucket, path,
gcsClient, profiler);
+return new CloudResettableInputStream(bufferedWriter, bufferProvider);
}
@Override
@@ -202,7 +205,7 @@
public boolean exists(String bucket, String path) {
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path,
Storage.BlobGetOption.fields(Storage.BlobField.values()));
-return blob != null;
+return blob != null && blob.exists();
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index cccd9ec..5c5641e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
@@ -21,9 +21,10 @@
import static
org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig.WRITE_BUFFER_SIZE;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
-import org.apache.asterix.cloud.clients.ICloudWriter;
+import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.asterix.cloud.clients.profiler.IRequestProfiler;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.logging.log4j.LogManager;
@@ -34,7 +35,7 @@
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
-public class GCSWriter implements ICloudWriter {
+public class GCSWriter implements ICloudBuf
