[
https://issues.apache.org/jira/browse/CLOUDSTACK-9538?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15684261#comment-15684261
]
ASF GitHub Bot commented on CLOUDSTACK-9538:
--------------------------------------------
Github user blueorangutan commented on the issue:
https://github.com/apache/cloudstack/pull/1710
<b>Trillian test result (tid-377)</b>
Environment: kvm-centos7 (x2), Advanced Networking with Mgmt server 7
Total time taken: 27282 seconds
Marvin logs:
https://github.com/blueorangutan/acs-prs/releases/download/trillian/pr1710-t377-kvm-centos7.zip
Test completed. 47 look ok, 1 have error(s)
Test | Result | Time (s) | Test File
--- | --- | --- | ---
test_02_redundant_VPC_default_routes | `Failure` | 870.27 |
test_vpc_redundant.py
ContextSuite context=TestVPCRedundancy>:teardown | `Error` | 518.77 |
test_vpc_redundant.py
test_01_vpc_site2site_vpn | Success | 185.42 | test_vpc_vpn.py
test_01_vpc_remote_access_vpn | Success | 66.14 | test_vpc_vpn.py
test_01_redundant_vpc_site2site_vpn | Success | 271.58 | test_vpc_vpn.py
test_02_VPC_default_routes | Success | 309.46 | test_vpc_router_nics.py
test_01_VPC_nics_after_destroy | Success | 656.73 | test_vpc_router_nics.py
test_05_rvpc_multi_tiers | Success | 513.66 | test_vpc_redundant.py
test_04_rvpc_network_garbage_collector_nics | Success | 1286.92 |
test_vpc_redundant.py
test_03_create_redundant_VPC_1tier_2VMs_2IPs_2PF_ACL_reboot_routers |
Success | 554.96 | test_vpc_redundant.py
test_01_create_redundant_VPC_2tiers_4VMs_4IPs_4PF_ACL | Success | 1303.03 |
test_vpc_redundant.py
test_09_delete_detached_volume | Success | 15.51 | test_volumes.py
test_08_resize_volume | Success | 15.43 | test_volumes.py
test_07_resize_fail | Success | 20.52 | test_volumes.py
test_06_download_detached_volume | Success | 15.31 | test_volumes.py
test_05_detach_volume | Success | 100.30 | test_volumes.py
test_04_delete_attached_volume | Success | 10.20 | test_volumes.py
test_03_download_attached_volume | Success | 15.41 | test_volumes.py
test_02_attach_volume | Success | 44.99 | test_volumes.py
test_01_create_volume | Success | 719.77 | test_volumes.py
test_deploy_vm_multiple | Success | 253.60 | test_vm_life_cycle.py
test_deploy_vm | Success | 0.03 | test_vm_life_cycle.py
test_advZoneVirtualRouter | Success | 0.03 | test_vm_life_cycle.py
test_10_attachAndDetach_iso | Success | 26.70 | test_vm_life_cycle.py
test_09_expunge_vm | Success | 125.25 | test_vm_life_cycle.py
test_08_migrate_vm | Success | 42.44 | test_vm_life_cycle.py
test_07_restore_vm | Success | 0.14 | test_vm_life_cycle.py
test_06_destroy_vm | Success | 125.92 | test_vm_life_cycle.py
test_03_reboot_vm | Success | 125.82 | test_vm_life_cycle.py
test_02_start_vm | Success | 10.17 | test_vm_life_cycle.py
test_01_stop_vm | Success | 40.33 | test_vm_life_cycle.py
test_CreateTemplateWithDuplicateName | Success | 131.03 | test_templates.py
test_08_list_system_templates | Success | 0.03 | test_templates.py
test_07_list_public_templates | Success | 0.04 | test_templates.py
test_05_template_permissions | Success | 0.06 | test_templates.py
test_04_extract_template | Success | 5.18 | test_templates.py
test_03_delete_template | Success | 5.11 | test_templates.py
test_02_edit_template | Success | 90.18 | test_templates.py
test_01_create_template | Success | 50.52 | test_templates.py
test_10_destroy_cpvm | Success | 161.65 | test_ssvm.py
test_09_destroy_ssvm | Success | 168.70 | test_ssvm.py
test_08_reboot_cpvm | Success | 131.69 | test_ssvm.py
test_07_reboot_ssvm | Success | 133.64 | test_ssvm.py
test_06_stop_cpvm | Success | 131.78 | test_ssvm.py
test_05_stop_ssvm | Success | 163.84 | test_ssvm.py
test_04_cpvm_internals | Success | 1.23 | test_ssvm.py
test_03_ssvm_internals | Success | 3.37 | test_ssvm.py
test_02_list_cpvm_vm | Success | 0.13 | test_ssvm.py
test_01_list_sec_storage_vm | Success | 0.14 | test_ssvm.py
test_01_snapshot_root_disk | Success | 16.34 | test_snapshots.py
test_04_change_offering_small | Success | 239.69 | test_service_offerings.py
test_03_delete_service_offering | Success | 0.04 | test_service_offerings.py
test_02_edit_service_offering | Success | 0.09 | test_service_offerings.py
test_01_create_service_offering | Success | 0.11 | test_service_offerings.py
test_02_sys_template_ready | Success | 0.13 | test_secondary_storage.py
test_01_sys_vm_start | Success | 0.19 | test_secondary_storage.py
test_09_reboot_router | Success | 40.33 | test_routers.py
test_08_start_router | Success | 30.40 | test_routers.py
test_07_stop_router | Success | 10.16 | test_routers.py
test_06_router_advanced | Success | 0.06 | test_routers.py
test_05_router_basic | Success | 0.04 | test_routers.py
test_04_restart_network_wo_cleanup | Success | 5.72 | test_routers.py
test_03_restart_network_cleanup | Success | 60.54 | test_routers.py
test_02_router_internal_adv | Success | 1.05 | test_routers.py
test_01_router_internal_basic | Success | 0.59 | test_routers.py
test_router_dns_guestipquery | Success | 77.23 | test_router_dns.py
test_router_dns_externalipquery | Success | 0.06 | test_router_dns.py
test_router_dhcphosts | Success | 328.57 | test_router_dhcphosts.py
test_01_updatevolumedetail | Success | 0.09 | test_resource_detail.py
test_01_reset_vm_on_reboot | Success | 130.98 | test_reset_vm_on_reboot.py
test_createRegion | Success | 0.04 | test_regions.py
test_create_pvlan_network | Success | 5.23 | test_pvlan.py
test_dedicatePublicIpRange | Success | 0.44 | test_public_ip_range.py
test_04_rvpc_privategw_static_routes | Success | 477.23 |
test_privategw_acl.py
test_03_vpc_privategw_restart_vpc_cleanup | Success | 491.65 |
test_privategw_acl.py
test_02_vpc_privategw_static_routes | Success | 427.20 |
test_privategw_acl.py
test_01_vpc_privategw_acl | Success | 98.40 | test_privategw_acl.py
test_01_primary_storage_nfs | Success | 35.99 | test_primary_storage.py
test_createPortablePublicIPRange | Success | 15.22 |
test_portable_publicip.py
test_createPortablePublicIPAcquire | Success | 15.46 |
test_portable_publicip.py
test_isolate_network_password_server | Success | 90.06 |
test_password_server.py
test_UpdateStorageOverProvisioningFactor | Success | 0.16 |
test_over_provisioning.py
test_oobm_zchange_password | Success | 20.52 | test_outofbandmanagement.py
test_oobm_multiple_mgmt_server_ownership | Success | 16.52 |
test_outofbandmanagement.py
test_oobm_issue_power_status | Success | 10.52 | test_outofbandmanagement.py
test_oobm_issue_power_soft | Success | 15.51 | test_outofbandmanagement.py
test_oobm_issue_power_reset | Success | 15.52 | test_outofbandmanagement.py
test_oobm_issue_power_on | Success | 16.52 | test_outofbandmanagement.py
test_oobm_issue_power_off | Success | 15.52 | test_outofbandmanagement.py
test_oobm_issue_power_cycle | Success | 15.52 | test_outofbandmanagement.py
test_oobm_enabledisable_across_clusterzones | Success | 62.57 |
test_outofbandmanagement.py
test_oobm_enable_feature_valid | Success | 5.20 |
test_outofbandmanagement.py
test_oobm_enable_feature_invalid | Success | 0.12 |
test_outofbandmanagement.py
test_oobm_disable_feature_valid | Success | 0.19 |
test_outofbandmanagement.py
test_oobm_disable_feature_invalid | Success | 5.20 |
test_outofbandmanagement.py
test_oobm_configure_invalid_driver | Success | 0.11 |
test_outofbandmanagement.py
test_oobm_configure_default_driver | Success | 0.10 |
test_outofbandmanagement.py
test_oobm_background_powerstate_sync | Success | 29.88 |
test_outofbandmanagement.py
test_extendPhysicalNetworkVlan | Success | 15.37 |
test_non_contigiousvlan.py
test_01_nic | Success | 636.79 | test_nic.py
test_releaseIP | Success | 279.82 | test_network.py
test_reboot_router | Success | 430.04 | test_network.py
test_public_ip_user_account | Success | 10.28 | test_network.py
test_public_ip_admin_account | Success | 40.29 | test_network.py
test_network_rules_acquired_public_ip_3_Load_Balancer_Rule | Success |
66.89 | test_network.py
test_network_rules_acquired_public_ip_2_nat_rule | Success | 61.91 |
test_network.py
test_network_rules_acquired_public_ip_1_static_nat_rule | Success | 124.25
| test_network.py
test_delete_account | Success | 274.01 | test_network.py
test_02_port_fwd_on_non_src_nat | Success | 55.65 | test_network.py
test_01_port_fwd_on_src_nat | Success | 111.81 | test_network.py
test_nic_secondaryip_add_remove | Success | 203.46 |
test_multipleips_per_nic.py
login_test_saml_user | Success | 24.68 | test_login.py
test_assign_and_removal_lb | Success | 133.61 | test_loadbalance.py
test_02_create_lb_rule_non_nat | Success | 187.34 | test_loadbalance.py
test_01_create_lb_rule_src_nat | Success | 232.12 | test_loadbalance.py
test_03_list_snapshots | Success | 0.08 | test_list_ids_parameter.py
test_02_list_templates | Success | 0.04 | test_list_ids_parameter.py
test_01_list_volumes | Success | 0.03 | test_list_ids_parameter.py
test_07_list_default_iso | Success | 0.06 | test_iso.py
test_05_iso_permissions | Success | 0.07 | test_iso.py
test_04_extract_Iso | Success | 5.22 | test_iso.py
test_03_delete_iso | Success | 95.17 | test_iso.py
test_02_edit_iso | Success | 0.06 | test_iso.py
test_01_create_iso | Success | 21.91 | test_iso.py
test_04_rvpc_internallb_haproxy_stats_on_all_interfaces | Success | 258.81
| test_internal_lb.py
test_03_vpc_internallb_haproxy_stats_on_all_interfaces | Success | 214.98 |
test_internal_lb.py
test_02_internallb_roundrobin_1RVPC_3VM_HTTP_port80 | Success | 531.17 |
test_internal_lb.py
test_01_internallb_roundrobin_1VPC_3VM_HTTP_port80 | Success | 455.57 |
test_internal_lb.py
test_dedicateGuestVlanRange | Success | 10.31 | test_guest_vlan_range.py
test_UpdateConfigParamWithScope | Success | 0.16 | test_global_settings.py
test_rolepermission_lifecycle_update | Success | 7.07 | test_dynamicroles.py
test_rolepermission_lifecycle_list | Success | 7.14 | test_dynamicroles.py
test_rolepermission_lifecycle_delete | Success | 6.81 | test_dynamicroles.py
test_rolepermission_lifecycle_create | Success | 6.79 | test_dynamicroles.py
test_rolepermission_lifecycle_concurrent_updates | Success | 7.04 |
test_dynamicroles.py
test_role_lifecycle_update_role_inuse | Success | 6.79 |
test_dynamicroles.py
test_role_lifecycle_update | Success | 6.93 | test_dynamicroles.py
test_role_lifecycle_list | Success | 6.92 | test_dynamicroles.py
test_role_lifecycle_delete | Success | 12.12 | test_dynamicroles.py
test_role_lifecycle_create | Success | 6.79 | test_dynamicroles.py
test_role_inuse_deletion | Success | 6.76 | test_dynamicroles.py
test_role_account_acls_multiple_mgmt_servers | Success | 9.21 |
test_dynamicroles.py
test_role_account_acls | Success | 9.27 | test_dynamicroles.py
test_default_role_deletion | Success | 6.88 | test_dynamicroles.py
test_04_create_fat_type_disk_offering | Success | 0.07 |
test_disk_offerings.py
test_03_delete_disk_offering | Success | 0.04 | test_disk_offerings.py
test_02_edit_disk_offering | Success | 0.05 | test_disk_offerings.py
test_02_create_sparse_type_disk_offering | Success | 0.07 |
test_disk_offerings.py
test_01_create_disk_offering | Success | 0.10 | test_disk_offerings.py
test_deployvm_userdispersing | Success | 30.97 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_userconcentrated | Success | 55.82 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_firstfit | Success | 55.66 |
test_deploy_vms_with_varied_deploymentplanners.py
test_deployvm_userdata_post | Success | 10.37 |
test_deploy_vm_with_userdata.py
test_deployvm_userdata | Success | 55.72 | test_deploy_vm_with_userdata.py
test_02_deploy_vm_root_resize | Success | 6.89 |
test_deploy_vm_root_resize.py
test_01_deploy_vm_root_resize | Success | 7.05 |
test_deploy_vm_root_resize.py
test_00_deploy_vm_root_resize | Success | 274.13 |
test_deploy_vm_root_resize.py
test_deploy_vm_from_iso | Success | 208.40 | test_deploy_vm_iso.py
test_DeployVmAntiAffinityGroup | Success | 61.56 | test_affinity_groups.py
test_03_delete_vm_snapshots | Skipped | 0.00 | test_vm_snapshots.py
test_02_revert_vm_snapshots | Skipped | 0.00 | test_vm_snapshots.py
test_01_test_vm_volume_snapshot | Skipped | 0.00 | test_vm_snapshots.py
test_01_create_vm_snapshots | Skipped | 0.00 | test_vm_snapshots.py
test_06_copy_template | Skipped | 0.00 | test_templates.py
test_static_role_account_acls | Skipped | 0.02 | test_staticroles.py
test_01_scale_vm | Skipped | 0.00 | test_scale_vm.py
test_01_primary_storage_iscsi | Skipped | 0.04 | test_primary_storage.py
test_06_copy_iso | Skipped | 0.00 | test_iso.py
test_deploy_vgpu_enabled_vm | Skipped | 0.01 |
test_deploy_vgpu_enabled_vm.py
> Deleting Snapshot From Primary Storage Fails on RBD Storage if you already
> delete vm's itself
> ---------------------------------------------------------------------------------------------
>
> Key: CLOUDSTACK-9538
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-9538
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Components: KVM, Snapshot, Storage Controller
> Affects Versions: 4.9.0
> Environment: Ubuntu 14.04 Management Server + Ubuntu 14.04 KVM
> Reporter: Özhan Rüzgar Karaman
>
> Hi;
> We plan to store vm snapshots as vm backups on secondary storage while we
> still destroyed/expunged related vm. The idea is good there was a bug which
> blocks this idea to work and it was fixed with CLOUDSTACK-9297 bug.
> Normally with 4.9 release we expected this idea to work on our on our 4.9 ACS
> environment but we noticed that because we are using rbd as primary storage
> we need to fix one minor bug for this idea to work.
> The problem occurs because CLOUDSTACK-8302 bug fixed on 4.9 release and it
> block our idea to work. If you destroy a vm which is on RBD Storage as
> primary storage it also deletes any related snapshots of that vm on Primary
> RBD Storage. So after vm destroy no disk file or snapshot file over RBD
> Storage. This is good for cleanup purposes on primary storage but
> XenserverSnapshotStrategy.deleteSnapshot class method did not expect this to
> happen.
> org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot
> method receives exception. The code tries 10 times on KVM node to remove RBD
> snapshot but because there is no snapshot on RBD side it get exception after
> 10 retries, it also spends nearly 5 minutes to delete snapshots and after
> that it ends with an error like "Failed to delete snapshot" error.
> I think we need to disable snapshot cleanup on primary storage only for RBD
> type Primary Storage if its related vm was already been destroyed. (Because
> vm destroy stage removed all snapshots related to vm on primary storage so
> there is no need to take any action on primary storage.)
> We make some tests below to make this issue clear for bug.
> 1) We create a vm with 3 snapshots on ACS.
> mysql> select * from snapshot_store_ref where snapshot_id in (93,94,95);
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-------+--------------+---------+---------------------+-----------+
> | id | store_id | snapshot_id | created | last_updated | job_id
> | store_role | size | physical_size | parent_snapshot_id |
> install_path
> | state | update_count | ref_cnt | updated | volume_id |
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-------+--------------+---------+---------------------+-----------+
> | 185 | 1 | 93 | 2016-10-12 10:13:44 | NULL | NULL
> | Primary | 28991029248 | 28991029248 | 0 |
> cst4/bb9ca3c7-96d6-4465-85b5-cd01f4d635f2/54008bf3-43dd-469d-91a7-4acd146d7b84
> | Ready | 2 | 0 | 2016-10-12 10:13:45 | 4774 |
> | 186 | 1 | 93 | 2016-10-12 10:13:45 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> | Ready | 2 | 0 | 2016-10-12 10:15:04 | 4774 |
> | 187 | 1 | 94 | 2016-10-12 10:15:38 | NULL | NULL
> | Primary | 28991029248 | 28991029248 | 0 |
> cst4/bb9ca3c7-96d6-4465-85b5-cd01f4d635f2/45fc4f44-b377-49c0-9264-5d813fefe93f
> | Ready | 2 | 0 | 2016-10-12 10:15:39 | 4774 |
> | 188 | 1 | 94 | 2016-10-12 10:15:39 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> | Ready | 2 | 0 | 2016-10-12 10:16:52 | 4774 |
> | 189 | 1 | 95 | 2016-10-12 10:17:08 | NULL | NULL
> | Primary | 28991029248 | 28991029248 | 0 |
> cst4/bb9ca3c7-96d6-4465-85b5-cd01f4d635f2/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> | Ready | 2 | 0 | 2016-10-12 10:17:09 | 4774 |
> | 190 | 1 | 95 | 2016-10-12 10:17:09 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> | Ready | 2 | 0 | 2016-10-12 10:18:23 | 4774 |
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-------+--------------+---------+---------------------+-----------+
> 6 rows in set (0.00 sec)
> mysql> select * from snapshots where name like '%TolgaC72%';
> +----+----------------+------------+-----------+-----------+------------------+----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> | id | data_center_id | account_id | domain_id | volume_id | disk_offering_id
> | status | path | name | uuid |
> snapshot_type | type_description | size | created |
> removed | backup_snap_id | swift_id | sechost_id | prev_snap_id |
> hypervisor_type | version | s3_id | min_iops | max_iops |
> +----+----------------+------------+-----------+-----------+------------------+----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> | 93 | 1 | 2 | 1 | 4774 | 19
> | BackedUp | NULL | S1TolgaC72Test41 | 0f84acd8-9a5c-4de5-a58a-b528b4e72839 |
> 0 | MANUAL | 28991029248 | 2016-10-12 10:13:42 | NULL
> | NULL | NULL | NULL | NULL | KVM |
> 2.2 | NULL | NULL | NULL |
> | 94 | 1 | 2 | 1 | 4774 | 19
> | BackedUp | NULL | S2TolgaC72Test41 | 51ef5246-27ae-435e-92ac-7428bab64cb8 |
> 0 | MANUAL | 28991029248 | 2016-10-12 10:15:36 | NULL
> | NULL | NULL | NULL | NULL | KVM |
> 2.2 | NULL | NULL | NULL |
> | 95 | 1 | 2 | 1 | 4774 | 19
> | BackedUp | NULL | S3TolgaC72Test41 | e5550dd6-9a43-4dd7-959a-7aabf26f0593 |
> 0 | MANUAL | 28991029248 | 2016-10-12 10:17:06 | NULL
> | NULL | NULL | NULL | NULL | KVM |
> 2.2 | NULL | NULL | NULL |
> +----+----------------+------------+-----------+-----------+------------------+----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> 3 rows in set (0.00 sec)
> mysql> select uuid, name from volumes where name='ROOT-4795';
> +--------------------------------------+-----------+
> | uuid | name |
> +--------------------------------------+-----------+
> | bb9ca3c7-96d6-4465-85b5-cd01f4d635f2 | ROOT-4795 |
> +--------------------------------------+-----------+
> 1 row in set (0.00 sec)
> mysql>
>
> #VM + 3 Snapshots
> #Ceph / Primary Storage
> root@mont1:/var/log/ceph# rbd ls -l -p cst4|grep
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2
> 27648M cst4/fb80c005-51d2-4fc4-8a98-39073866e826@cloudstack-base-snap 2
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2@54008bf3-43dd-469d-91a7-4acd146d7b84
> 27648M cst4/fb80c005-51d2-4fc4-8a98-39073866e826@cloudstack-base-snap 2
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2@45fc4f44-b377-49c0-9264-5d813fefe93f
> 27648M cst4/fb80c005-51d2-4fc4-8a98-39073866e826@cloudstack-base-snap 2
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2@1c1890bf-5a55-4060-bf3f-cf064629dcdf
> 27648M cst4/fb80c005-51d2-4fc4-8a98-39073866e826@cloudstack-base-snap 2
> root@mont1:/var/log/ceph#
> #NFS / Secondary Side
> root@cloudstackt4:/CST/S# ls -l
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:18
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:16
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:15
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> root@cloudstackt4:/CST/S#
> 2) We destroy only vm not snapshots so 3 snapshots are active with BackedUp
> Status. On vm destroy stage vm destroy method removes all vm's files and its
> related snapshots on primary storage.
> #VM Destroyed we have 3 Snapshots on BackedUp state
> #Ceph Side / Primary Storage
> root@mont1:/var/log/ceph# rbd ls -l -p cst4|grep
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2
> root@mont1:/var/log/ceph#
> #NFS / Secondary Side
> root@cloudstackt4:/CST/S# ls -l
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:18
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:16
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:15
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> root@cloudstackt4:/CST/S#
> 3) We destroy 1 Snapshot named S3TolgaC72Test41
> #Ceph Side / Primary Storage
> root@mont1:/var/log/ceph# rbd ls -l -p cst4|grep
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2
> root@mont1:/var/log/ceph#
> #NFS / Secondary Side
> root@cloudstackt4:/CST/S# ls -l
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> ls: cannot access snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf: No
> such file or directory
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:16
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> -rw-r--r-- 1 root root 28991029248 Oct 12 13:15
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> #SQL Side
> mysql> select * from snapshots where name like '%TolgaC72%' and name !=
> 'StolgaC72Revert';
> +----+----------------+------------+-----------+-----------+------------------+-----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> | id | data_center_id | account_id | domain_id | volume_id | disk_offering_id
> | status | path | name | uuid
> | snapshot_type | type_description | size | created |
> removed | backup_snap_id | swift_id | sechost_id | prev_snap_id |
> hypervisor_type | version | s3_id | min_iops | max_iops |
> +----+----------------+------------+-----------+-----------+------------------+-----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> | 93 | 1 | 2 | 1 | 4774 | 19
> | BackedUp | NULL | S1TolgaC72Test41 | 0f84acd8-9a5c-4de5-a58a-b528b4e72839
> | 0 | MANUAL | 28991029248 | 2016-10-12 10:13:42 | NULL
> | NULL | NULL | NULL | NULL | KVM
> | 2.2 | NULL | NULL | NULL |
> | 94 | 1 | 2 | 1 | 4774 | 19
> | BackedUp | NULL | S2TolgaC72Test41 | 51ef5246-27ae-435e-92ac-7428bab64cb8
> | 0 | MANUAL | 28991029248 | 2016-10-12 10:15:36 | NULL
> | NULL | NULL | NULL | NULL | KVM
> | 2.2 | NULL | NULL | NULL |
> | 95 | 1 | 2 | 1 | 4774 | 19
> | Destroyed | NULL | S3TolgaC72Test41 | e5550dd6-9a43-4dd7-959a-7aabf26f0593
> | 0 | MANUAL | 28991029248 | 2016-10-12 10:17:06 | NULL
> | NULL | NULL | NULL | NULL | KVM
> | 2.2 | NULL | NULL | NULL |
> +----+----------------+------------+-----------+-----------+------------------+-----------+------+------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+----------+----------+
> 3 rows in set (0.00 sec)
> mysql> select * from snapshot_store_ref where snapshot_id in (93,94,95);
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-----------+--------------+---------+---------------------+-----------+
> | id | store_id | snapshot_id | created | last_updated | job_id
> | store_role | size | physical_size | parent_snapshot_id |
> install_path
> | state | update_count | ref_cnt | updated | volume_id |
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-----------+--------------+---------+---------------------+-----------+
> | 185 | 1 | 93 | 2016-10-12 10:13:44 | NULL | NULL
> | Primary | 28991029248 | 28991029248 | 0 |
> cst4/bb9ca3c7-96d6-4465-85b5-cd01f4d635f2/54008bf3-43dd-469d-91a7-4acd146d7b84
> | Ready | 2 | 0 | 2016-10-12 10:13:45 | 4774 |
> | 186 | 1 | 93 | 2016-10-12 10:13:45 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/54008bf3-43dd-469d-91a7-4acd146d7b84
> | Ready | 2 | 0 | 2016-10-12 10:15:04 | 4774 |
> | 187 | 1 | 94 | 2016-10-12 10:15:38 | NULL | NULL
> | Primary | 28991029248 | 28991029248 | 0 |
> cst4/bb9ca3c7-96d6-4465-85b5-cd01f4d635f2/45fc4f44-b377-49c0-9264-5d813fefe93f
> | Ready | 2 | 0 | 2016-10-12 10:15:39 | 4774 |
> | 188 | 1 | 94 | 2016-10-12 10:15:39 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/45fc4f44-b377-49c0-9264-5d813fefe93f
> | Ready | 2 | 0 | 2016-10-12 10:16:52 | 4774 |
> | 190 | 1 | 95 | 2016-10-12 10:17:09 | NULL | NULL
> | Image | 28991029248 | 28991029248 | 0 |
> snapshots/2/4774/1c1890bf-5a55-4060-bf3f-cf064629dcdf
> | Destroyed | 4 | 0 | 2016-10-12 10:48:31 | 4774 |
> +-----+----------+-------------+---------------------+--------------+--------+------------+-------------+---------------+--------------------+--------------------------------------------------------------------------------+-----------+--------------+---------+---------------------+-----------+
> 5 rows in set (0.00 sec)
> mysql>
> ERROR on Cloudstack Management Server: Failed to delete snapshot
> 2016-10-12 13:54:02,911 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy]
> (API-Job-Executor-70:ctx-06858bab job-34505 ctx-a6bc52fb) (logid:a2b92099)
> Failed to delete snapshot:
> com.cloud.utils.exception.CloudRuntimeException: Failed to remove snapshot
> at
> org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.deleteSnapshot(SnapshotServiceImpl.java:396)
> at
> org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot(XenserverSnapshotStrategy.java:272)
> at
> com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshot(SnapshotManagerImpl.java:454)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> at
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> at
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> at
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> at
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> at com.sun.proxy.$Proxy192.deleteSnapshot(Unknown Source)
> at
> org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd.execute(DeleteSnapshotCmd.java:103)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:150)
> at
> com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:554)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> at
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> at
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> at
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:502)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> 2016-10-12 13:54:02,918 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
> (API-Job-Executor-70:ctx-06858bab job-34505) (logid:a2b92099) Complete async
> job-34505, jobStatus: FAILED, resultCode: 530,
> result:
> org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"Failed
> to delete snapshot"}
> ERROR on KVM Agent
> 2016-10-12 13:49:38,227 DEBUG [kvm.storage.LibvirtStorageAdaptor]
> (agentRequest-Handler-3:null) Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2'
> 2016-10-12 13:49:38,227 DEBUG [kvm.storage.LibvirtStorageAdaptor]
> (agentRequest-Handler-3:null) Refreshing storage pool
> 01b375db-d3f5-33c1-9389-8bf226c887e8
> 2016-10-12 13:49:41,253 DEBUG [kvm.storage.KVMStoragePoolManager]
> (agentRequest-Handler-3:null) Failed to find
> volume:bb9ca3c7-96d6-4465-85b5-cd01f4d635f2 due
> tocom.cloud.utils.exception.CloudRuntimeException: Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2', retry:2
> 2016-10-12 13:51:53,819 DEBUG [kvm.storage.KVMStoragePoolManager]
> (agentRequest-Handler-3:null) Failed to find
> volume:bb9ca3c7-96d6-4465-85b5-cd01f4d635f2 due
> tocom.cloud.utils.exception.CloudRuntimeException: Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2', retry:6
> 2016-10-12 13:52:26,858 DEBUG [kvm.storage.KVMStoragePoolManager]
> (agentRequest-Handler-3:null) Failed to find
> volume:bb9ca3c7-96d6-4465-85b5-cd01f4d635f2 due
> tocom.cloud.utils.exception.CloudRuntimeException: Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2', retry:7
> 2016-10-12 13:53:33,390 DEBUG [kvm.storage.KVMStoragePoolManager]
> (agentRequest-Handler-3:null) Failed to find
> volume:bb9ca3c7-96d6-4465-85b5-cd01f4d635f2 due
> tocom.cloud.utils.exception.CloudRuntimeException: Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2', retry:9
> 2016-10-12 13:54:03,390 ERROR [kvm.storage.KVMStorageProcessor]
> (agentRequest-Handler-3:null) Failed to remove snapshot , with exception:
> com.cloud.utils.exception.CloudRuntimeException:
> com.cloud.utils.exception.CloudRuntimeException: Could not find volume
> bb9ca3c7-96d6-4465-85b5-cd01f4d635f2: Storage volume not found: no storage
> vol with matching name 'bb9ca3c7-96d6-4465-85b5-cd01f4d635f2'
> Thanks
> Özhan
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
