>From Savyasach Reddy <savyasach.enuko...@couchbase.com>: Savyasach Reddy has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 )
Change subject: [ASTERIXDB-3390][STO]: Update storage scheme for GCS ...................................................................... [ASTERIXDB-3390][STO]: Update storage scheme for GCS Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd --- M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java 4 files changed, 23 insertions(+), 19 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/72/18272/1 diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf index d0ebd24..ccb7f9d 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf @@ -64,7 +64,7 @@ cloud.deployment=true storage.buffercache.pagesize=32KB storage.partitioning=static -cloud.storage.scheme=gcs +cloud.storage.scheme=gs cloud.storage.bucket=cloud-storage-container cloud.storage.region=us-west2 cloud.storage.endpoint=http://127.0.0.1:4443 diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java index 35ab467..d5f508d 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java @@ -27,7 +27,7 @@ public class CloudClientProvider { private static final String S3 = "s3"; - private static final String GCS = "gcs"; + private static final String GCS = "gs"; private CloudClientProvider() { throw new AssertionError("do not instantiate"); diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java index c725ca5..0bee4e9 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java @@ -32,7 +32,9 @@ import java.util.List; import java.util.Set; +import org.apache.asterix.cloud.CloudResettableInputStream; import org.apache.asterix.cloud.IWriteBufferProvider; +import org.apache.asterix.cloud.clients.ICloudBufferedWriter; import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.clients.ICloudWriter; import org.apache.asterix.cloud.clients.IParallelDownloader; @@ -87,7 +89,8 @@ @Override public ICloudWriter createdWriter(String bucket, String path, IWriteBufferProvider bufferProvider) { - return new GCSWriter(bucket, path, gcsClient, profiler); + ICloudBufferedWriter bufferedWriter = new GCSWriter(bucket, path, gcsClient, profiler); + return new CloudResettableInputStream(bufferedWriter, bufferProvider); } @Override @@ -202,7 +205,7 @@ public boolean exists(String bucket, String path) { profiler.objectGet(); Blob blob = gcsClient.get(bucket, path, Storage.BlobGetOption.fields(Storage.BlobField.values())); - return blob != null; + return blob != null && blob.exists(); } @Override diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java index cccd9ec..5c5641e 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java @@ -21,9 +21,10 @@ import static org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig.WRITE_BUFFER_SIZE; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; -import org.apache.asterix.cloud.clients.ICloudWriter; +import org.apache.asterix.cloud.clients.ICloudBufferedWriter; import org.apache.asterix.cloud.clients.profiler.IRequestProfiler; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.logging.log4j.LogManager; @@ -34,7 +35,7 @@ import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; -public class GCSWriter implements ICloudWriter { +public class GCSWriter implements ICloudBufferedWriter { private static final Logger LOGGER = LogManager.getLogger(); private final String bucket; private final String path; @@ -51,16 +52,12 @@ } @Override - public int write(ByteBuffer header, ByteBuffer page) throws HyracksDataException { - return write(header) + write(page); - } - - @Override - public int write(ByteBuffer page) throws HyracksDataException { + public int upload(InputStream stream, int length) throws HyracksDataException { profiler.objectMultipartUpload(); setUploadId(); int written = 0; try { + ByteBuffer page = ByteBuffer.wrap(stream.readNBytes(length)); while (page.hasRemaining()) { written += writer.write(page); } @@ -72,13 +69,8 @@ } @Override - public int write(byte[] b, int off, int len) throws HyracksDataException { - return write(ByteBuffer.wrap(b, off, len)); - } - - @Override - public void write(int b) throws HyracksDataException { - write(ByteBuffer.wrap(new byte[] { (byte) b })); + public boolean isEmpty() { + return !uploadStarted; } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18272 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I2415275efa5b0fbb9bd93306b4fb140f4b2e59cd Gerrit-Change-Number: 18272 Gerrit-PatchSet: 1 Gerrit-Owner: Savyasach Reddy <savyasach.enuko...@couchbase.com> Gerrit-MessageType: newchange
