openinx commented on a change in pull request #3553: URL: https://github.com/apache/iceberg/pull/3553#discussion_r749056121
########## File path: aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.aliyun.oss; + +import com.aliyun.oss.OSS; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.iceberg.aliyun.AliyunClientFactory; +import org.apache.iceberg.aliyun.AliyunProperties; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.util.SerializableSupplier; + +/** + * FileIO implementation backend by OSS. Review comment: Nit: `backend` -> `backed`. ########## File path: aliyun/src/main/java/org/apache/iceberg/aliyun/DefaultAliyunClientFactory.java ########## @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.aliyun; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +public class DefaultAliyunClientFactory implements AliyunClientFactory { + private AliyunProperties aliyunProperties; + + @Override + public OSS oss() { + Preconditions.checkNotNull(aliyunProperties, + "Cannot create aliyun oss client before initializing the AliyunClientFactory."); Review comment: Nit: we usually use 4 spaces to indent the new line ( rather than 8 spaces). ########## File path: aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java ########## @@ -33,14 +34,20 @@ private final String bucketName = OSS_TEST_RULE.testBucketName(); private final String keyPrefix = OSS_TEST_RULE.keyPrefix(); + private OSSFileIO fileIO; Review comment: Why not move the OSSFileIO initialization to [here](https://github.com/apache/iceberg/pull/3553/files#diff-39d04fa07c6ca88cd781acc081580ce62cd460d19327e748e6995cefd073873fR51), because in my view, it's recommend to initialize the OSSFileIO by using the catalog properties rather than the OSSFIleIO constructor(for the most of the cases). ########## File path: aliyun/src/main/java/org/apache/iceberg/aliyun/AliyunClientFactory.java ########## @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.aliyun; + +import com.aliyun.oss.OSS; +import java.io.Serializable; +import java.util.Map; +import org.apache.iceberg.common.DynConstructors; + +public interface AliyunClientFactory extends Serializable { + /** + * Create an aliyun OSS client. + * + * @return oss client. + */ + OSS oss(); + + /** + * Initialize Aliyun client factory from catalog properties. + * + * @param properties catalog properties + */ + void initialize(Map<String, String> properties); + + /** + * Returns an initialized {@link AliyunProperties} + */ + AliyunProperties aliyunProperties(); + + static AliyunClientFactory load(Map<String, String> properties) { + String impl = properties.getOrDefault("client.factory", DefaultAliyunClientFactory.class.getName()); Review comment: I think we will need to provide the KMS approach to manage the access key and access secret in future. https://github.com/aliyun/aliyun-sdk-managed-credentials-providers-java ########## File path: aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java ########## @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.aliyun.oss; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSClientBuilder; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.io.ByteStreams; +import org.apache.iceberg.util.SerializableSupplier; +import org.apache.iceberg.util.SerializationUtil; +import org.junit.Assert; +import org.junit.Test; + +public class TestOSSFileIO extends AliyunOSSTestBase { + private static final String OSS_IMPL_CLASS = OSSFileIO.class.getName(); + + private final OSS ossClient = ossClient().get(); + private final Random random = ThreadLocalRandom.current(); + private final Configuration conf = new Configuration(); + + @Test + public void testOutputFile() throws IOException { + String location = randomLocation(); + int dataSize = 1024 * 10; + byte[] data = randomData(dataSize); + + OutputFile out = fileIO().newOutputFile(location); + try (OutputStream os = out.create(); InputStream is = new ByteArrayInputStream(data)) { + ByteStreams.copy(is, os); + } Review comment: Could we abstract this common logic into a small method to write the memory bytes to remote oss object ? I have saw lots of the places are implementing the same logic. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
