harikrishna-patnala commented on code in PR #7752: URL: https://github.com/apache/cloudstack/pull/7752#discussion_r1274717934
########## api/src/main/java/org/apache/cloudstack/api/BaseCmd.java: ########## @@ -29,6 +29,7 @@ import javax.inject.Inject; +import com.cloud.storage.object.BucketApiService; Review Comment: reorder this please ########## api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java: ########## @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.ObjectStore; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = UpdateObjectStoragePoolCmd.APINAME, description = "Updates object storage pool", responseObject = ObjectStoreResponse.class, entityType = {ObjectStore.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0") +public class UpdateObjectStoragePoolCmd extends BaseCmd { Review Comment: extra tab here in the class ########## api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java: ########## @@ -22,6 +22,8 @@ import java.util.Map; import java.util.Set; +import com.cloud.storage.Bucket; Review Comment: reorder this please ########## api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java: ########## @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.storage.Bucket; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "listBuckets", description = "Lists all Buckets.", responseObject = BucketResponse.class, responseView = ResponseView.Restricted, entityType = { Review Comment: since = "4.19.0" is missing here ########## api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java: ########## @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Bucket; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "createBucket", responseObject = BucketResponse.class, Review Comment: since = "4.19.0" is missing here ########## api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java: ########## @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.storage.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteBucket", description = "Deletes an empty Bucket.", responseObject = SuccessResponse.class, entityType = {Bucket.class}, Review Comment: since = "4.19.0" is missing here ########## ui/public/locales/en.json: ########## @@ -2242,6 +2248,18 @@ "label.zonenamelabel": "Zone name", "label.zones": "Zones", "label.zonewizard.traffictype.storage": "Storage: Traffic between primary and secondary storage servers, such as VM templates and snapshots.", +"label.buckets": "Buckets", +"label.objectstorageid": "Object Storage Pool", +"label.bucket.update": "Update Bucket", +"label.bucket.delete": "Delete Bucket", +"label.quota": "Quota", Review Comment: Quota is already there I think, please check ########## server/src/main/java/com/cloud/api/ApiResponseHelper.java: ########## @@ -37,6 +37,9 @@ import javax.inject.Inject; +import com.cloud.storage.Bucket; Review Comment: reorder this please ########## api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java: ########## @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.storage.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "updateBucket", description = "Updates Bucket properties", responseObject = SuccessResponse.class, entityType = {Bucket.class}, Review Comment: since = "4.19.0" is missing here ########## plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java: ########## @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.storage.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.BucketExistsArgs; +import io.minio.DeleteBucketEncryptionArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.SetBucketEncryptionArgs; +import io.minio.SetBucketPolicyArgs; +import io.minio.SetBucketVersioningArgs; +import io.minio.admin.MinioAdminClient; +import io.minio.admin.QuotaUnit; +import io.minio.admin.UserInfo; +import io.minio.admin.messages.DataUsageInfo; +import io.minio.messages.SseConfiguration; +import io.minio.messages.VersioningConfiguration; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.inject.Inject; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String MINIO_ACCESS_KEY = "minio-accesskey"; + private static final String MINIO_SECRET_KEY = "minio-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + //ToDo Client pool mgmt + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + MinioClient minioClient = getMinIOClient(storeId); + Account account = _accountDao.findById(accountId); + try { + if(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket already exists with name "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(objectLock).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + + String policy = " {\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Action\": \"s3:*\",\n" + + " \"Effect\": \"Allow\",\n" + + " \"Principal\": \"*\",\n" + + " \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n" + + " }\n" + + " ],\n" + + " \"Version\": \"2012-10-17\"\n" + + " }"; + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String policyName = "acs-"+account.getAccountName()+"-policy"; + String userName = "acs-"+account.getAccountName(); + try { + minioAdminClient.addCannedPolicy(policyName, policy); + minioAdminClient.setPolicy(userName, false, policyName); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + String accessKey = _accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List<Bucket> listBuckets(long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + List<Bucket> bucketsList = new ArrayList<>(); + try { + List<io.minio.messages.Bucket> minIOBuckets = minioClient.listBuckets(); + for(io.minio.messages.Bucket minIObucket : minIOBuckets) { + Bucket bucket = new BucketObject(); + bucket.setName(minIObucket.name()); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket doesn't exist: "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + //ToDo: check bucket empty + try { + minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + + String publicPolicy = builder.toString(); + + //ToDo Support custom policy + String policyConfig = (policy.equalsIgnoreCase("public"))? publicPolicy : privatePolicy; + + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketPolicy( + SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String accessKey = "acs-"+account.getAccountName(); + // Check user exists + try { + UserInfo userInfo = minioAdminClient.getUserInfo(accessKey); + if(userInfo != null) { + s_logger.debug("User already exists in MinIO store: "+accessKey); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user: "+accessKey); + } + + KeyGenerator generator = null; + try { + generator = KeyGenerator.getInstance("HmacSHA1"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException(e); + } + SecretKey key = generator.generateKey(); + String secretKey = Base64.encodeBase64URLSafeString(key.getEncoded()); + try { + minioAdminClient.addUser(accessKey, UserInfo.Status.ENABLED, secretKey, "", new ArrayList<String>()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + // Store user credentials + Map<String, String> details = new HashMap<>(); + details.put(MINIO_ACCESS_KEY, accessKey); + details.put(MINIO_SECRET_KEY, secretKey); + _accountDetailsDao.persist(accountId, details); + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() + .bucket(bucketName) + .config(SseConfiguration.newConfigWithSseS3Rule()) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() + .bucket(bucketName) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + /* + public void setBucketPolicy(String bucketName, long storeId, String policy) { Review Comment: Do we need to delete this ? ########## ui/src/config/section/storage.js: ########## @@ -443,54 +443,37 @@ export default { ] }, { - name: 'backup', Review Comment: We need backup also right! may be by mistake. Can you keep this and add new sections for buckets ########## plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java: ########## @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.storage.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.BucketExistsArgs; +import io.minio.DeleteBucketEncryptionArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.SetBucketEncryptionArgs; +import io.minio.SetBucketPolicyArgs; +import io.minio.SetBucketVersioningArgs; +import io.minio.admin.MinioAdminClient; +import io.minio.admin.QuotaUnit; +import io.minio.admin.UserInfo; +import io.minio.admin.messages.DataUsageInfo; +import io.minio.messages.SseConfiguration; +import io.minio.messages.VersioningConfiguration; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.inject.Inject; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String MINIO_ACCESS_KEY = "minio-accesskey"; + private static final String MINIO_SECRET_KEY = "minio-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + //ToDo Client pool mgmt + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + MinioClient minioClient = getMinIOClient(storeId); + Account account = _accountDao.findById(accountId); + try { + if(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket already exists with name "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(objectLock).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + + String policy = " {\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Action\": \"s3:*\",\n" + + " \"Effect\": \"Allow\",\n" + + " \"Principal\": \"*\",\n" + + " \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n" + + " }\n" + + " ],\n" + + " \"Version\": \"2012-10-17\"\n" + + " }"; + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String policyName = "acs-"+account.getAccountName()+"-policy"; + String userName = "acs-"+account.getAccountName(); + try { + minioAdminClient.addCannedPolicy(policyName, policy); + minioAdminClient.setPolicy(userName, false, policyName); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + String accessKey = _accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List<Bucket> listBuckets(long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + List<Bucket> bucketsList = new ArrayList<>(); + try { + List<io.minio.messages.Bucket> minIOBuckets = minioClient.listBuckets(); + for(io.minio.messages.Bucket minIObucket : minIOBuckets) { + Bucket bucket = new BucketObject(); + bucket.setName(minIObucket.name()); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket doesn't exist: "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + //ToDo: check bucket empty + try { + minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + + String publicPolicy = builder.toString(); + + //ToDo Support custom policy + String policyConfig = (policy.equalsIgnoreCase("public"))? publicPolicy : privatePolicy; + + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketPolicy( + SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String accessKey = "acs-"+account.getAccountName(); + // Check user exists + try { + UserInfo userInfo = minioAdminClient.getUserInfo(accessKey); + if(userInfo != null) { + s_logger.debug("User already exists in MinIO store: "+accessKey); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user: "+accessKey); + } + + KeyGenerator generator = null; + try { + generator = KeyGenerator.getInstance("HmacSHA1"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException(e); + } + SecretKey key = generator.generateKey(); + String secretKey = Base64.encodeBase64URLSafeString(key.getEncoded()); + try { + minioAdminClient.addUser(accessKey, UserInfo.Status.ENABLED, secretKey, "", new ArrayList<String>()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + // Store user credentials + Map<String, String> details = new HashMap<>(); + details.put(MINIO_ACCESS_KEY, accessKey); + details.put(MINIO_SECRET_KEY, secretKey); + _accountDetailsDao.persist(accountId, details); + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() + .bucket(bucketName) + .config(SseConfiguration.newConfigWithSseS3Rule()) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() + .bucket(bucketName) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + /* + public void setBucketPolicy(String bucketName, long storeId, String policy) { + + String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + + String publicPolicy = builder.toString(); + //Default to private policy + //ToDo Support custom policy + String policyConfig = privatePolicy; + + if(policy.equals("public")) { + policyConfig = publicPolicy; + } + + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketPolicy( + SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build()); + + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } */ + + @Override Review Comment: Can you please add null checks for all the return objects in the below methods. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
