[ https://issues.apache.org/jira/browse/CLOUDSTACK-9867?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16305224#comment-16305224 ]
ASF GitHub Bot commented on CLOUDSTACK-9867: -------------------------------------------- rhtyd closed pull request #2035: CLOUDSTACK-9867:VM snapshot on primary storage usage metrics URL: https://github.com/apache/cloudstack/pull/2035 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/api/src/com/cloud/event/EventTypes.java b/api/src/com/cloud/event/EventTypes.java index ce410a6795d..26b692205bf 100644 --- a/api/src/com/cloud/event/EventTypes.java +++ b/api/src/com/cloud/event/EventTypes.java @@ -241,6 +241,8 @@ // Snapshots public static final String EVENT_SNAPSHOT_CREATE = "SNAPSHOT.CREATE"; + public static final String EVENT_SNAPSHOT_ON_PRIMARY = "SNAPSHOT.ON_PRIMARY"; + public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY"; public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE"; public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT"; public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE"; @@ -462,6 +464,8 @@ // vm snapshot events public static final String EVENT_VM_SNAPSHOT_CREATE = "VMSNAPSHOT.CREATE"; public static final String EVENT_VM_SNAPSHOT_DELETE = "VMSNAPSHOT.DELETE"; + public static final String EVENT_VM_SNAPSHOT_ON_PRIMARY = "VMSNAPSHOT.ON_PRIMARY"; + public static final String EVENT_VM_SNAPSHOT_OFF_PRIMARY = "VMSNAPSHOT.OFF_PRIMARY"; public static final String EVENT_VM_SNAPSHOT_REVERT = "VMSNAPSHOT.REVERTTO"; // external network device events @@ -711,6 +715,8 @@ // Snapshots entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class); entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class); + entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class); + entityEventDetails.put(EVENT_SNAPSHOT_OFF_PRIMARY, Snapshot.class); entityEventDetails.put(EVENT_SNAPSHOT_POLICY_CREATE, SnapshotPolicy.class); entityEventDetails.put(EVENT_SNAPSHOT_POLICY_UPDATE, SnapshotPolicy.class); entityEventDetails.put(EVENT_SNAPSHOT_POLICY_DELETE, SnapshotPolicy.class); diff --git a/api/src/org/apache/cloudstack/usage/UsageTypes.java b/api/src/org/apache/cloudstack/usage/UsageTypes.java index 08cd7ddb721..f03d9a9f733 100644 --- a/api/src/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/org/apache/cloudstack/usage/UsageTypes.java @@ -43,6 +43,7 @@ public static final int VM_DISK_BYTES_WRITE = 24; public static final int VM_SNAPSHOT = 25; public static final int VOLUME_SECONDARY = 26; + public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static List<UsageTypeResponse> listUsageTypes() { List<UsageTypeResponse> responseList = new ArrayList<UsageTypeResponse>(); @@ -65,6 +66,7 @@ responseList.add(new UsageTypeResponse(VM_DISK_BYTES_READ, "VM Disk usage(Bytes Read)")); responseList.add(new UsageTypeResponse(VM_DISK_BYTES_WRITE, "VM Disk usage(Bytes Write)")); responseList.add(new UsageTypeResponse(VM_SNAPSHOT, "VM Snapshot storage usage")); + responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); return responseList; } } diff --git a/engine/schema/resources/META-INF/db/schema-41000to41100.sql b/engine/schema/resources/META-INF/db/schema-41000to41100.sql index 3dbe5c3e564..661a5939a5c 100644 --- a/engine/schema/resources/META-INF/db/schema-41000to41100.sql +++ b/engine/schema/resources/META-INF/db/schema-41000to41100.sql @@ -490,6 +490,23 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.2.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.1.0' AND guest_os_id not in (1,2,3,4,56,101,56,58,93,94,50,51,87,88,89,90,91,92,26,27,28,29,40,41,42,43,44,45,96,97,107,108,109,110,151,152,153); +-- Add table to track primary storage in use for snapshots +DROP TABLE IF EXISTS `cloud_usage`.`usage_snapshot_on_primary`; +CREATE TABLE `cloud_usage`.`usage_snapshot_on_primary` ( + `id` bigint(20) unsigned NOT NULL, + `zone_id` bigint(20) unsigned NOT NULL, + `account_id` bigint(20) unsigned NOT NULL, + `domain_id` bigint(20) unsigned NOT NULL, + `vm_id` bigint(20) unsigned NOT NULL, + `name` varchar(128), + `type` int(1) unsigned NOT NULL, + `physicalsize` bigint(20), + `virtualsize` bigint(20), + `created` datetime NOT NULL, + `deleted` datetime, + INDEX `i_usage_snapshot_on_primary` (`account_id`,`id`,`vm_id`,`created`) +) ENGINE=InnoDB CHARSET=utf8; + -- Change monitor patch for apache2 in systemvm UPDATE `cloud`.`monitoring_services` SET pidfile="/var/run/apache2/apache2.pid" WHERE process_name="apache2" AND service_name="apache2"; diff --git a/engine/schema/src/com/cloud/usage/UsageSnapshotOnPrimaryVO.java b/engine/schema/src/com/cloud/usage/UsageSnapshotOnPrimaryVO.java new file mode 100644 index 00000000000..74f944b9ece --- /dev/null +++ b/engine/schema/src/com/cloud/usage/UsageSnapshotOnPrimaryVO.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "usage_snapshot_on_primary") +public class UsageSnapshotOnPrimaryVO implements InternalIdentity { + + @Column(name = "id") + // volumeId + private long id; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "vm_id") + private long vmId; + + @Column(name = "name") + private String name; + + @Column(name = "type") + private int snapshotType; + + @Column(name = "physicalsize") + private long physicalSize; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created = null; + + @Column(name = "deleted") + @Temporal(value = TemporalType.TIMESTAMP) + private Date deleted; + + @Column(name = "virtualsize") + private Long virtualSize; + + protected UsageSnapshotOnPrimaryVO() { + } + + public UsageSnapshotOnPrimaryVO(long id, long zoneId, long accountId, long domainId, long vmId, String name, int type, long virtualSize, long physicalSize, Date created, Date deleted) { + this.id = id; + this.zoneId = zoneId; + this.accountId = accountId; + this.domainId = domainId; + this.snapshotType = type; + this.physicalSize = physicalSize; + this.virtualSize = virtualSize; + this.created = created; + this.vmId = vmId; + this.name = name; + this.deleted = deleted; + } + + public long getZoneId() { + return zoneId; + } + + public long getAccountId() { + return accountId; + } + + public long getDomainId() { + return domainId; + } + + public int getSnapshotType() { + return snapshotType; + } + + public long getPhysicalSize() { + return physicalSize; + } + + public Long getVirtualSize() { + return virtualSize; + } + + public Date getDeleted() { + return deleted; + } + + public void setDeleted(Date deleted) { + this.deleted = deleted; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public long getVmId() { + return vmId; + } + + public String getName() { + return name; + } + + @Override + public long getId() { + return this.id; + } + + @Override + public String toString() { + return "UsageSnapshotOnPrimaryVO [id=" + id + ", zoneId=" + zoneId + ", accountId=" + accountId + ", domainId=" + domainId + ", vmId=" + vmId + ", name=" + name + + ", snapshotType=" + snapshotType + ", physicalSize=" + physicalSize + ", created=" + created + ", deleted=" + deleted + ", virtualSize=" + virtualSize + "]"; + } + +} diff --git a/engine/schema/src/com/cloud/usage/UsageVMSnapshotVO.java b/engine/schema/src/com/cloud/usage/UsageVMSnapshotVO.java index 1266d638fe6..05a1b29c6f3 100644 --- a/engine/schema/src/com/cloud/usage/UsageVMSnapshotVO.java +++ b/engine/schema/src/com/cloud/usage/UsageVMSnapshotVO.java @@ -120,4 +120,10 @@ public long getId() { return this.id; } + @Override + public String toString() { + return "UsageVMSnapshotVO [id=" + id + ", zoneId=" + zoneId + ", accountId=" + accountId + ", domainId=" + domainId + ", vmId=" + vmId + ", diskOfferingId=" + + diskOfferingId + ", size=" + size + ", created=" + created + ", processed=" + processed + "]"; + } + } diff --git a/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDao.java b/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDao.java new file mode 100644 index 00000000000..09d6e00cf9a --- /dev/null +++ b/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import java.util.Date; +import java.util.List; + +import com.cloud.usage.UsageSnapshotOnPrimaryVO; +import com.cloud.utils.db.GenericDao; + +public interface UsageVMSnapshotOnPrimaryDao extends GenericDao<UsageSnapshotOnPrimaryVO, Long> { + + public void updateDeleted(UsageSnapshotOnPrimaryVO usage); + + public List<UsageSnapshotOnPrimaryVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate); + +} diff --git a/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDaoImpl.java b/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDaoImpl.java new file mode 100644 index 00000000000..59269387008 --- /dev/null +++ b/engine/schema/src/com/cloud/usage/dao/UsageVMSnapshotOnPrimaryDaoImpl.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.usage.dao; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.usage.UsageSnapshotOnPrimaryVO; +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.TransactionLegacy; + +@Component +public class UsageVMSnapshotOnPrimaryDaoImpl extends GenericDaoBase<UsageSnapshotOnPrimaryVO, Long> implements UsageVMSnapshotOnPrimaryDao { + public static final Logger s_logger = Logger.getLogger(UsageVMSnapshotOnPrimaryDaoImpl.class.getName()); + protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, zone_id, account_id, domain_id, vm_id, name, type, physicalsize, virtualsize, created, deleted " + + " FROM usage_snapshot_on_primary" + " WHERE account_id = ? " + " AND ( (created < ? AND deleted is NULL)" + + " OR ( deleted BETWEEN ? AND ?)) ORDER BY created asc"; + protected static final String UPDATE_DELETED = "UPDATE usage_snapshot_on_primary SET deleted = ? WHERE account_id = ? AND id = ? and vm_id = ? and created = ?"; + + @Override + public void updateDeleted(UsageSnapshotOnPrimaryVO usage) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + PreparedStatement pstmt = null; + try { + txn.start(); + pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED); + pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted())); + pstmt.setLong(2, usage.getAccountId()); + pstmt.setLong(3, usage.getId()); + pstmt.setLong(4, usage.getVmId()); + pstmt.setString(5, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getCreated())); + pstmt.executeUpdate(); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Error updating UsageSnapshotOnPrimaryVO", e); + } finally { + txn.close(); + } + } + + @Override + public List<UsageSnapshotOnPrimaryVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate) { + List<UsageSnapshotOnPrimaryVO> usageRecords = new ArrayList<UsageSnapshotOnPrimaryVO>(); + + String sql = GET_USAGE_RECORDS_BY_ACCOUNT; + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + PreparedStatement pstmt = null; + + try { + int i = 1; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(i++, accountId); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + s_logger.debug("GET_USAGE_RECORDS_BY_ACCOUNT " + pstmt); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + //id, zone_id, account_id, domain_iVMSnapshotVOd, vm_id, disk_offering_id, size, created, deleted + Long vId = Long.valueOf(rs.getLong(1)); + Long zoneId = Long.valueOf(rs.getLong(2)); + Long acctId = Long.valueOf(rs.getLong(3)); + Long dId = Long.valueOf(rs.getLong(4)); + Long vmId = Long.valueOf(rs.getLong(5)); + String name = String.valueOf(rs.getString(6)); + Integer type = Integer.valueOf(rs.getInt(7)); + Long physicalSize = Long.valueOf(rs.getLong(8)); + Long virtaulSize = Long.valueOf(rs.getLong(9)); + Date createdDate = null; + Date deleteDate = null; + String createdTS = rs.getString(10); + String deleted = rs.getString(11); + + if (createdTS != null) { + createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); + } + if (deleted != null) { + deleteDate = DateUtil.parseDateString(s_gmtTimeZone, deleted); + } + usageRecords.add(new UsageSnapshotOnPrimaryVO(vId, zoneId, acctId, dId, vmId, name, type, virtaulSize, physicalSize, createdDate, deleteDate)); + } + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Error getting usage records", e); + } finally { + txn.close(); + } + + return usageRecords; + } + +} diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index b7bc4475d16..601959bdcbd 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -49,6 +49,8 @@ import org.apache.log4j.Logger; import com.cloud.storage.CreateSnapshotPayload; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -216,6 +218,8 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), + snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); return result; } catch (InterruptedException e) { s_logger.debug("Failed to create snapshot", e); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index 185cf567fa0..7aa538845bc 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -19,6 +19,9 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.to.DiskTO; import com.cloud.dc.dao.ClusterDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -147,6 +150,7 @@ public boolean deleteSnapshot(Long snapshotId) { * @return true if snapshot is removed, false otherwise */ + @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_OFF_PRIMARY, eventDescription = "deleting snapshot", async = true) private boolean cleanupSnapshotOnPrimaryStore(long snapshotId) { SnapshotObject snapshotObj = (SnapshotObject)snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Primary); @@ -176,6 +180,8 @@ private boolean cleanupSnapshotOnPrimaryStore(long snapshotId) { snapshotSvr.deleteSnapshot(snapshotObj); snapshotObj.processEvent(Snapshot.Event.OperationSucceeded); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_OFF_PRIMARY, snapshotObj.getAccountId(), snapshotObj.getDataCenterId(), snapshotId, + snapshotObj.getName(), null, null, 0L, snapshotObj.getClass().getName(), snapshotObj.getUuid()); } catch (Exception e) { s_logger.debug("Failed to delete snapshot: ", e); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index a673a462375..837b20150ab 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -42,6 +42,8 @@ import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -97,7 +99,6 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotInfo parentSnapshot = snapshot.getParent(); if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { - s_logger.debug("backup an empty snapshot"); // don't need to backup this snapshot SnapshotDataStoreVO parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); if (parentSnapshotOnBackupStore != null && parentSnapshotOnBackupStore.getState() == State.Ready) { @@ -254,7 +255,6 @@ public boolean deleteSnapshot(Long snapshotId) { } if (snapshotVO.getState() == Snapshot.State.CreatedOnPrimary) { - s_logger.debug("delete snapshot on primary storage:"); snapshotVO.setState(Snapshot.State.Destroyed); snapshotDao.update(snapshotId, snapshotVO); return true; @@ -428,6 +428,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { SnapshotDataStoreVO snapshotDataStoreVO = snapshotStoreDao.findByStoreSnapshot(primaryStore.getRole(), primaryStore.getId(), parentSnapshotId); if (snapshotDataStoreVO != null) { parentSnapshotId = snapshotDataStoreVO.getParentSnapshotId(); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_OFF_PRIMARY, parent.getAccountId(), parent.getDataCenterId(), parent.getId(), + parent.getName(), null, null, 0L, 0L, parent.getClass().getName(), parent.getUuid()); snapshotStoreDao.remove(snapshotDataStoreVO.getId()); } else { parentSnapshotId = null; diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 71a5e104d31..ebe8b27b207 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -119,6 +119,14 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(userVm.getId()); + long prev_chain_size = 0; + long virtual_size=0; + for (VolumeObjectTO volume : volumeTOs) { + virtual_size += volume.getSize(); + VolumeVO volumeVO = volumeDao.findById(volume.getId()); + prev_chain_size += volumeVO.getVmSnapshotChainSize() == null ? 0 : volumeVO.getVmSnapshotChainSize(); + } + VMSnapshotTO current = null; VMSnapshotVO currentSnapshot = vmSnapshotDao.findCurrentSnapshotByVmId(userVm.getId()); if (currentSnapshot != null) @@ -150,10 +158,12 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { processAnswer(vmSnapshotVO, userVm, answer, hostId); s_logger.debug("Create vm snapshot " + vmSnapshot.getName() + " succeeded for vm: " + userVm.getInstanceName()); result = true; - + long new_chain_size=0; for (VolumeObjectTO volumeTo : answer.getVolumeTOs()) { publishUsageEvent(EventTypes.EVENT_VM_SNAPSHOT_CREATE, vmSnapshot, userVm, volumeTo); + new_chain_size += volumeTo.getSize(); } + publishUsageEvent(EventTypes.EVENT_VM_SNAPSHOT_ON_PRIMARY, vmSnapshot, userVm, new_chain_size - prev_chain_size, virtual_size); return vmSnapshot; } else { String errMsg = "Creating VM snapshot: " + vmSnapshot.getName() + " failed"; @@ -208,9 +218,12 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { if (answer != null && answer.getResult()) { DeleteVMSnapshotAnswer deleteVMSnapshotAnswer = (DeleteVMSnapshotAnswer)answer; processAnswer(vmSnapshotVO, userVm, answer, hostId); + long full_chain_size=0; for (VolumeObjectTO volumeTo : deleteVMSnapshotAnswer.getVolumeTOs()) { publishUsageEvent(EventTypes.EVENT_VM_SNAPSHOT_DELETE, vmSnapshot, userVm, volumeTo); + full_chain_size += volumeTo.getSize(); } + publishUsageEvent(EventTypes.EVENT_VM_SNAPSHOT_OFF_PRIMARY, vmSnapshot, userVm, full_chain_size, 0L); return true; } else { String errMsg = (answer == null) ? null : answer.getDetails(); @@ -325,7 +338,16 @@ private void publishUsageEvent(String type, VMSnapshot vmSnapshot, UserVm userVm } } UsageEventUtils.publishUsageEvent(type, vmSnapshot.getAccountId(), userVm.getDataCenterId(), userVm.getId(), vmSnapshot.getName(), offeringId, volume.getId(), // save volume's id into templateId field - volumeTo.getSize(), VMSnapshot.class.getName(), vmSnapshot.getUuid()); + volumeTo.getSize(), VMSnapshot.class.getName(), vmSnapshot.getUuid()); + } + + private void publishUsageEvent(String type, VMSnapshot vmSnapshot, UserVm userVm, Long vmSnapSize, Long virtualSize) { + try { + UsageEventUtils.publishUsageEvent(type, vmSnapshot.getAccountId(), userVm.getDataCenterId(), userVm.getId(), vmSnapshot.getName(), 0L, 0L, vmSnapSize, virtualSize, + VMSnapshot.class.getName(), vmSnapshot.getUuid()); + } catch (Exception e) { + s_logger.error("Failed to publis usage event " + type, e); + } } @Override diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index c25c1adaf7a..158ee18f911 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -209,6 +209,7 @@ public EndPoint select(DataObject srcData, DataObject destData) { @Override public EndPoint select(DataObject srcData, DataObject destData, StorageAction action) { + s_logger.error("IR24 select BACKUPSNAPSHOT from primary to secondary " + srcData.getId() + " dest=" + destData.getId()); if (action == StorageAction.BACKUPSNAPSHOT && srcData.getDataStore().getRole() == DataStoreRole.Primary) { SnapshotInfo srcSnapshot = (SnapshotInfo)srcData; VolumeInfo volumeInfo = srcSnapshot.getBaseVolume(); diff --git a/framework/quota/src/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/org/apache/cloudstack/quota/constant/QuotaTypes.java index 6e1432d848d..97e22da111a 100644 --- a/framework/quota/src/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -55,6 +55,7 @@ quotaTypeList.put(VM_DISK_BYTES_READ, new QuotaTypes(VM_DISK_BYTES_READ, "VM_DISK_BYTES_READ", "GB", "VM Disk usage(Bytes Read)")); quotaTypeList.put(VM_DISK_BYTES_WRITE, new QuotaTypes(VM_DISK_BYTES_WRITE, "VPN_USERS", "GB", "VM Disk usage(Bytes Write)")); quotaTypeList.put(VM_SNAPSHOT, new QuotaTypes(VM_SNAPSHOT, "VM_SNAPSHOT", "GB-Month", "VM Snapshot storage usage")); + quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", "GB-Month", "VM Snapshot primary storage usage")); quotaTypeList.put(CPU_CLOCK_RATE, new QuotaTypes(CPU_CLOCK_RATE, "CPU_CLOCK_RATE", "Compute-Month", "Quota tariff for using 1 CPU of clock rate 100MHz")); quotaTypeList.put(CPU_NUMBER, new QuotaTypes(CPU_NUMBER, "CPU_NUMBER", "Compute-Month", "Quota tariff for running VM that has 1vCPU")); quotaTypeList.put(MEMORY, new QuotaTypes(MEMORY, "MEMORY", "Compute-Month", "Quota tariff for using 1MB of RAM")); diff --git a/usage/src/com/cloud/usage/UsageManagerImpl.java b/usage/src/com/cloud/usage/UsageManagerImpl.java index 4a60b24a0f0..d70910249b2 100644 --- a/usage/src/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/com/cloud/usage/UsageManagerImpl.java @@ -57,6 +57,7 @@ import com.cloud.usage.dao.UsageNetworkOfferingDao; import com.cloud.usage.dao.UsagePortForwardingRuleDao; import com.cloud.usage.dao.UsageSecurityGroupDao; +import com.cloud.usage.dao.UsageVMSnapshotOnPrimaryDao; import com.cloud.usage.dao.UsageStorageDao; import com.cloud.usage.dao.UsageVMInstanceDao; import com.cloud.usage.dao.UsageVMSnapshotDao; @@ -75,6 +76,7 @@ import com.cloud.usage.parser.VPNUserUsageParser; import com.cloud.usage.parser.VmDiskUsageParser; import com.cloud.usage.parser.VolumeUsageParser; +import com.cloud.usage.parser.VMSanpshotOnPrimaryParser; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -87,6 +89,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; @@ -145,6 +148,8 @@ @Inject private UsageVMSnapshotDao _usageVMSnapshotDao; @Inject + private UsageVMSnapshotOnPrimaryDao _usageSnapshotOnPrimaryDao; + @Inject private QuotaManager _quotaManager; @Inject private QuotaAlertManager _alertManager; @@ -939,6 +944,18 @@ private boolean parseHelperTables(AccountVO account, Date currentStartDate, Date s_logger.debug("VM Snapshot usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); } } + parsed = VMSnapshotUsageParser.parse(account, currentStartDate, currentEndDate); + if (s_logger.isDebugEnabled()) { + if (!parsed) { + s_logger.debug("VM Snapshot usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); + } + } + parsed = VMSanpshotOnPrimaryParser.parse(account, currentStartDate, currentEndDate); + if (s_logger.isDebugEnabled()) { + if (!parsed) { + s_logger.debug("VM Snapshot on primary usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); + } + } return parsed; } @@ -968,6 +985,8 @@ private void createHelperRecord(UsageEventVO event) { createSecurityGroupEvent(event); } else if (isVmSnapshotEvent(eventType)) { createVMSnapshotEvent(event); + } else if (isVmSnapshotOnPrimaryEvent(eventType)) { + createVmSnapshotOnPrimaryEvent(event); } } @@ -1043,6 +1062,12 @@ private boolean isVmSnapshotEvent(String eventType) { return (eventType.equals(EventTypes.EVENT_VM_SNAPSHOT_CREATE) || eventType.equals(EventTypes.EVENT_VM_SNAPSHOT_DELETE)); } + private boolean isVmSnapshotOnPrimaryEvent(String eventType) { + if (eventType == null) + return false; + return (eventType.equals(EventTypes.EVENT_VM_SNAPSHOT_ON_PRIMARY) || eventType.equals(EventTypes.EVENT_VM_SNAPSHOT_OFF_PRIMARY)); + } + private void createVMHelperEvent(UsageEventVO event) { // One record for handling VM.START and VM.STOP @@ -1806,6 +1831,43 @@ private void createVMSnapshotEvent(UsageEventVO event) { _usageVMSnapshotDao.persist(vsVO); } + private void createVmSnapshotOnPrimaryEvent(UsageEventVO event) { + Long vmId = event.getResourceId(); + String name = event.getResourceName(); + if (EventTypes.EVENT_VM_SNAPSHOT_ON_PRIMARY.equals(event.getType())) { + Long zoneId = event.getZoneId(); + Long accountId = event.getAccountId(); + long physicalsize = (event.getSize() == null) ? 0 : event.getSize(); + long virtualsize = (event.getVirtualSize() == null) ? 0 : event.getVirtualSize(); + Date created = event.getCreateDate(); + Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); + Long domainId = acct.getDomainId(); + UsageSnapshotOnPrimaryVO vsVO = new UsageSnapshotOnPrimaryVO(vmId, zoneId, accountId, domainId, vmId, name, 0, virtualsize, physicalsize, created, null); + if (s_logger.isDebugEnabled()) { + s_logger.debug("createSnapshotOnPrimaryEvent UsageSnapshotOnPrimaryVO " + vsVO); + } + _usageSnapshotOnPrimaryDao.persist(vsVO); + } else if (EventTypes.EVENT_VM_SNAPSHOT_OFF_PRIMARY.equals(event.getType())) { + QueryBuilder<UsageSnapshotOnPrimaryVO> sc = QueryBuilder.create(UsageSnapshotOnPrimaryVO.class); + sc.and(sc.entity().getAccountId(), SearchCriteria.Op.EQ, event.getAccountId()); + sc.and(sc.entity().getId(), SearchCriteria.Op.EQ, vmId); + sc.and(sc.entity().getName(), SearchCriteria.Op.EQ, name); + sc.and(sc.entity().getDeleted(), SearchCriteria.Op.NULL); + List<UsageSnapshotOnPrimaryVO> vmsnaps = sc.list(); + if (vmsnaps.size() > 1) { + s_logger.warn("More that one usage entry for vm snapshot: " + name + " for vm id:" + vmId + " assigned to account: " + event.getAccountId() + + "; marking them all as deleted..."); + } + for (UsageSnapshotOnPrimaryVO vmsnap : vmsnaps) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("deleting vm snapshot name: " + vmsnap.getName() + " from account: " + vmsnap.getAccountId()); + } + vmsnap.setDeleted(event.getCreateDate()); // there really shouldn't be more than one + _usageSnapshotOnPrimaryDao.updateDeleted(vmsnap); + } + } + } + private class Heartbeat extends ManagedContextRunnable { @Override protected void runInContext() { diff --git a/usage/src/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java b/usage/src/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java new file mode 100644 index 00000000000..851929507af --- /dev/null +++ b/usage/src/com/cloud/usage/parser/VMSanpshotOnPrimaryParser.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.parser; + +import java.text.DecimalFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import org.apache.cloudstack.usage.UsageTypes; + +import com.cloud.usage.UsageSnapshotOnPrimaryVO; +import com.cloud.usage.UsageVO; +import com.cloud.usage.dao.UsageDao; +import com.cloud.usage.dao.UsageVMSnapshotOnPrimaryDao; +import com.cloud.user.AccountVO; + +@Component +public class VMSanpshotOnPrimaryParser { + public static final Logger s_logger = Logger.getLogger(VMSanpshotOnPrimaryParser.class.getName()); + + private static UsageDao s_usageDao; + private static UsageVMSnapshotOnPrimaryDao s_usageSnapshotOnPrimaryDao; + + @Inject + private UsageDao _usageDao; + @Inject + private UsageVMSnapshotOnPrimaryDao _usageSnapshotOnPrimaryDao; + + @PostConstruct + void init() { + s_usageDao = _usageDao; + s_usageSnapshotOnPrimaryDao = _usageSnapshotOnPrimaryDao; + } + + public static boolean parse(AccountVO account, Date startDate, Date endDate) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Parsing all VmSnapshot on primary usage events for account: " + account.getId()); + } + if ((endDate == null) || endDate.after(new Date())) { + endDate = new Date(); + } + + List<UsageSnapshotOnPrimaryVO> usageUsageVMSnapshots = s_usageSnapshotOnPrimaryDao.getUsageRecords(account.getId(), account.getDomainId(), startDate, endDate); + + if (usageUsageVMSnapshots.isEmpty()) { + s_logger.debug("No VM snapshot on primary usage events for this period"); + return true; + } + + Map<String, UsageSnapshotOnPrimaryVO> unprocessedUsage = new HashMap<String, UsageSnapshotOnPrimaryVO>(); + for (UsageSnapshotOnPrimaryVO usageRec : usageUsageVMSnapshots) { + s_logger.debug("usageRec for VMsnap on primary " + usageRec.toString()); + String key = usageRec.getName(); + if (usageRec.getPhysicalSize() == 0) { + usageRec.setDeleted(new Date()); + s_usageSnapshotOnPrimaryDao.updateDeleted(usageRec); + } else { + unprocessedUsage.put(key, usageRec); + } + } + + for (String key : unprocessedUsage.keySet()) { + UsageSnapshotOnPrimaryVO usageRec = unprocessedUsage.get(key); + Date created = usageRec.getCreated(); + if (created.before(startDate)) { + created = startDate; + } + Date endDateEffective = endDate; + if (usageRec.getDeleted() != null && usageRec.getDeleted().before(endDate)){ + endDateEffective = usageRec.getDeleted(); + s_logger.debug("Remoevd vm snapshot found endDateEffective " + endDateEffective + " period end data " + endDate); + } + long duration = (endDateEffective.getTime() - created.getTime()) + 1; + createUsageRecord(UsageTypes.VM_SNAPSHOT_ON_PRIMARY, duration, created, endDateEffective, account, usageRec.getId(), usageRec.getName(), usageRec.getZoneId(), + usageRec.getVirtualSize(), usageRec.getPhysicalSize()); + } + + return true; + } + + private static void createUsageRecord(int usageType, long runningTime, Date startDate, Date endDate, AccountVO account, long vmId, String name, long zoneId, long virtualSize, + long physicalSize) { + // Our smallest increment is hourly for now + if (s_logger.isDebugEnabled()) { + s_logger.debug("Total running time " + runningTime + "ms"); + } + + float usage = runningTime / 1000f / 60f / 60f; + + DecimalFormat dFormat = new DecimalFormat("#.######"); + String usageDisplay = dFormat.format(usage); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Creating VMSnapshot On Primary usage record for vm: " + vmId + ", usage: " + usageDisplay + ", startDate: " + startDate + ", endDate: " + endDate + + ", for account: " + account.getId()); + } + + // Create the usage record + String usageDesc = "VMSnapshot On Primary Usage: " + "VM Id: " + vmId; + usageDesc += " Size: " + virtualSize; + + UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", usageType, new Double(usage), vmId, name, null, null, + vmId, physicalSize, virtualSize, startDate, endDate); + s_usageDao.persist(usageRecord); + } + +} ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > VM snapshots - not charged for the primary storage they use up > -------------------------------------------------------------- > > Key: CLOUDSTACK-9867 > URL: https://issues.apache.org/jira/browse/CLOUDSTACK-9867 > Project: CloudStack > Issue Type: Improvement > Security Level: Public(Anyone can view this level - this is the > default.) > Affects Versions: 4.10.0.0 > Reporter: Abhinandan Prateek > Assignee: Abhinandan Prateek > Fix For: Future > > > Currently, when a user takes a VM snapshot, the snapshot is kept on primary > storage. CloudStack has no mechanism to record the additional storage used by > these snapshots. > To add an additional usage metric that is “primary storage used by > snapshots”. -- This message was sent by Atlassian JIRA (v6.4.14#64029)