CLOUDSTACK-6092: Storage OverProvisioning as a Per Primary Basis Allow storage.overprovisioning.factor to be specified at storape pool level.
Signed-off-by: Sateesh Chodapuneedi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/48f8a95b Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/48f8a95b Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/48f8a95b Branch: refs/heads/resize-root Commit: 48f8a95b06b0348ba1349cb5434183c2c18710db Parents: 6a4927f Author: Saksham Srivastava <[email protected]> Authored: Tue Mar 11 17:52:00 2014 +0530 Committer: Sateesh Chodapuneedi <[email protected]> Committed: Fri Mar 14 12:16:46 2014 +0530 ---------------------------------------------------------------------- .../api/response/StoragePoolResponse.java | 8 ++ .../src/com/cloud/capacity/CapacityManager.java | 2 +- .../api/query/dao/StoragePoolJoinDaoImpl.java | 4 +- .../cloud/capacity/StorageCapacityListener.java | 2 +- .../com/cloud/storage/StorageManagerImpl.java | 25 +++-- setup/db/db/schema-430to440.sql | 3 + .../integration/smoke/test_over_provisioning.py | 107 +++++++++++++++++++ 7 files changed, 141 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java ---------------------------------------------------------------------- diff --git a/api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java index 03a4f34..98c90e2 100644 --- a/api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -107,6 +107,10 @@ public class StoragePoolResponse extends BaseResponse { @Param(description = "the scope of the storage pool") private String scope; + @SerializedName("overprovisionfactor") + @Param(description = "the overprovisionfactor for the storage pool") + private String overprovisionfactor; + @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "the hypervisor type of the storage pool") private String hypervisor; @@ -301,4 +305,8 @@ public class StoragePoolResponse extends BaseResponse { public void setSuitableForMigration(Boolean suitableForMigration) { this.suitableForMigration = suitableForMigration; } + + public void setOverProvisionFactor(String overProvisionFactor) { + this.overprovisionfactor = overProvisionFactor; + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/engine/components-api/src/com/cloud/capacity/CapacityManager.java ---------------------------------------------------------------------- diff --git a/engine/components-api/src/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/com/cloud/capacity/CapacityManager.java index bd1a610..038a2c0 100755 --- a/engine/components-api/src/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/com/cloud/capacity/CapacityManager.java @@ -44,7 +44,7 @@ public interface CapacityManager { "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, ConfigKey.Scope.Zone); static final ConfigKey<Double> StorageOverprovisioningFactor = new ConfigKey<Double>("Storage", Double.class, StorageOverprovisioningFactorCK, "2", - "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.Zone); + "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.StoragePool); static final ConfigKey<Double> StorageAllocatedCapacityDisableThreshold = new ConfigKey<Double>( "Alert", http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 274bf1c..1d89b19 100644 --- a/server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -24,13 +24,13 @@ import javax.inject.Inject; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityManager; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePool; import com.cloud.storage.StorageStats; @@ -103,6 +103,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase<StoragePoolJoinVO, Lo poolResponse.setClusterId(pool.getClusterUuid()); poolResponse.setClusterName(pool.getClusterName()); poolResponse.setTags(pool.getTag()); + poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId()))); // set async job if (pool.getJobId() != null) { @@ -156,6 +157,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase<StoragePoolJoinVO, Lo poolResponse.setDiskSizeTotal(pool.getCapacityBytes()); poolResponse.setDiskSizeAllocated(allocatedSize); poolResponse.setCapacityIops(pool.getCapacityIops()); + poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId()))); // TODO: StatsCollector does not persist data StorageStats stats = ApiDBUtils.getStoragePoolStatistics(pool.getId()); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/server/src/com/cloud/capacity/StorageCapacityListener.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/capacity/StorageCapacityListener.java b/server/src/com/cloud/capacity/StorageCapacityListener.java index 4322ecf..f83cbd3 100755 --- a/server/src/com/cloud/capacity/StorageCapacityListener.java +++ b/server/src/com/cloud/capacity/StorageCapacityListener.java @@ -67,7 +67,7 @@ public class StorageCapacityListener implements Listener { StartupStorageCommand ssCmd = (StartupStorageCommand)startup; if (ssCmd.getResourceType() == Storage.StorageResourceType.STORAGE_HOST) { - BigDecimal overProvFactor = _storageMgr.getStorageOverProvisioningFactor(server.getDataCenterId()); + BigDecimal overProvFactor = BigDecimal.valueOf(CapacityManager.StorageOverprovisioningFactor.value()); CapacityVO capacity = new CapacityVO(server.getId(), server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, (overProvFactor.multiply(new BigDecimal( server.getTotalSize()))).longValue(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/server/src/com/cloud/storage/StorageManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 913dc23..3b29b33 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -854,8 +854,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } @Override - public BigDecimal getStorageOverProvisioningFactor(Long dcId) { - return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(dcId)); + public BigDecimal getStorageOverProvisioningFactor(Long poolId) { + return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId)); } @Override @@ -869,13 +869,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C long totalOverProvCapacity; if (storagePool.getPoolType() == StoragePoolType.NetworkFilesystem || storagePool.getPoolType() == StoragePoolType.VMFS) { - BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getDataCenterId()); - totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue(); // All this is for the inaccuracy of floats for big number multiplication. + BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId()); + totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue(); + s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString() + " with overprovisioning factor " + + overProvFactor.toString()); + s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + storagePool.getCapacityBytes()); } else { + s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString()); totalOverProvCapacity = storagePool.getCapacityBytes(); } + s_logger.debug("Total over provisioned capacity of the pool " + storagePool.getName() + " id: " + storagePool.getId() + " is " + totalOverProvCapacity); if (capacities.size() == 0) { CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), allocated, totalOverProvCapacity, @@ -1552,16 +1557,22 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C long totalOverProvCapacity; if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.VMFS) { - totalOverProvCapacity = getStorageOverProvisioningFactor(pool.getDataCenterId()).multiply(new BigDecimal(pool.getCapacityBytes())).longValue(); + BigDecimal overProvFactor = getStorageOverProvisioningFactor(pool.getId()); + totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue(); + s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString() + " with overprovisioning factor " + + overProvFactor.toString()); + s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + pool.getCapacityBytes()); } else { totalOverProvCapacity = pool.getCapacityBytes(); + s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString()); } + s_logger.debug("Total capacity of the pool " + poolVO.getName() + " id: " + pool.getId() + " is " + totalOverProvCapacity); double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Checking pool: " + pool.getId() + " for volume allocation " + volumes.toString() + ", maxSize : " + totalOverProvCapacity + - ", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " + - storageAllocatedThreshold); + ", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " + + storageAllocatedThreshold); } double usedPercentage = (allocatedSizeWithtemplate + totalAskingSize) / (double)(totalOverProvCapacity); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/setup/db/db/schema-430to440.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-430to440.sql b/setup/db/db/schema-430to440.sql index 9298f72..5c5511a 100644 --- a/setup/db/db/schema-430to440.sql +++ b/setup/db/db/schema-430to440.sql @@ -742,3 +742,6 @@ UPDATE `cloud`.`guest_os` SET `created` = now(); ALTER TABLE `cloud`.`vm_reservation` ADD COLUMN `deployment_planner` varchar(40) DEFAULT NULL COMMENT 'Preferred deployment planner for the vm'; ALTER TABLE `cloud`.`vpc_offerings` ADD COLUMN supports_distributed_router boolean default false; ALTER TABLE `cloud`.`vpc` ADD COLUMN uses_distributed_router boolean default false; + +INSERT INTO `cloud`.`storage_pool_details` (pool_id,name,value,display) SELECT storage_pool.id,data_center_details.name,data_center_details.value,data_center_details.display FROM `cloud`.`storage_pool` JOIN `cloud`.`data_center_details` ON data_center_details.dc_id=storage_pool.data_center_id WHERE data_center_details.name = "storage.overprovisioning.factor"; +DELETE FROM `cloud`.`data_center_details` WHERE name="storage.overprovisioning.factor"; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/48f8a95b/test/integration/smoke/test_over_provisioning.py ---------------------------------------------------------------------- diff --git a/test/integration/smoke/test_over_provisioning.py b/test/integration/smoke/test_over_provisioning.py new file mode 100644 index 0000000..28f32b7af --- /dev/null +++ b/test/integration/smoke/test_over_provisioning.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Test for storage.overprovisioning.factor update +""" +#Import Local Modules +from marvin.cloudstackTestCase import * +from marvin.cloudstackAPI import * +from marvin.integration.lib.utils import * +from marvin.integration.lib.base import * +from marvin.integration.lib.common import * +from nose.plugins.attrib import attr +#Import System modules + +class TestUpdateOverProvision(cloudstackTestCase): + """ + Test to update a storage.overprovisioning.factor + """ + def setUp(self): + self.apiClient = self.testClient.getApiClient() + + @attr(tags=["simulator", "devcloud", "basic", "advanced"]) + def test_UpdateStorageOverProvisioningFactor(self): + """ + test update configuration setting at storage scope + @return: + """ + + """ 1. list storagepools for id """ + """ 2. list overprovisioning factor for storage pool """ + """ 3. update setting for the pool""" + """ 4. list overprovisioning factor for storage pool and assert""" + + """ list storagepools """ + storage_pools = StoragePool.list( + self.apiClient, + listall=True + ) + self.assertEqual( + isinstance(storage_pools, list), + True, + "List storage pools should not return empty response" + ) + + if len(storage_pools) < 1: + raise self.skipTest( + "The environment don't have storage pools required for test") + + for pool in storage_pools: + if pool.type == "NetworkFilesystem" or pool.type == "VMFS": + break + if pool.type != "NetworkFilesystem" and pool.type != "VMFS": + raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools") + + self.poolId = pool.id + """ list overprovisioning factor for storage pool """ + factorOld = float(pool.overprovisionfactor) + factorNew = str(factorOld + 1.0) + + """ update setting for the pool""" + updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() + updateConfigurationCmd.name = "storage.overprovisioning.factor" + updateConfigurationCmd.value = factorNew + updateConfigurationCmd.storageid = pool.id + + updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) + + self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value)) + + storage_pools = StoragePool.list( + self.apiClient, + id = self.poolId + ) + pool = storage_pools[0] + factorNew = float(pool.overprovisionfactor) + self.assertNotEqual(int(factorNew), int(factorOld)," Check if overprovision factor of storage pool has changed") + self.assertEqual(int(factorNew), int(factorOld + 1.0)," Check if overprovision factor of storage pool has increased by 1") + + def tearDown(self): + """Reset the storage.overprovisioning.factor back to its original value + @return: + """ + storage_pools = StoragePool.list( + self.apiClient, + id = self.poolId + ) + pool = storage_pools[0] + updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() + updateConfigurationCmd.name = "storage.overprovisioning.factor" + factorOld = float(pool.overprovisionfactor) + factorNew = str(factorOld - 1.0) + updateConfigurationCmd.value = factorNew + updateConfigurationCmd.storageid = pool.id + updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd)
