This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch 4.19
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit e2f187912cd5cf97d4ffe7fcdcdc003721be495d
Author: Abhishek Kumar <abhishek.mr...@gmail.com>
AuthorDate: Fri Feb 21 11:16:22 2025 +0530
cks: create separate service account in project
A separate service account will be created and added in the project, if
not exist already, when a Kubernetes cluster is deployed in a project.
This account will have a role with limited API access.
Cleanup clusters on owner account cleanup, delete service account
if needed
When the owner account of k8s clusters is deleted, while its node VMs
get expunged, the cluster entry in DB remain present. This fixes the
issue by cleaning up all clusters for the account deleted.
Project k8s service account will be deleted on account cleanup or when
there is no active k8s cluster remaining
Signed-off-by: Abhishek Kumar <abhishek.mr...@gmail.com>
---
.../cluster/KubernetesServiceHelper.java | 2 +
.../main/java/com/cloud/user/dao/AccountDao.java | 8 +-
.../java/com/cloud/user/dao/AccountDaoImpl.java | 27 ++-
.../cluster/KubernetesClusterManagerImpl.java | 251 ++++++++++++++++++---
.../cluster/KubernetesClusterService.java | 8 +-
.../cluster/KubernetesServiceHelperImpl.java | 8 +
.../cluster/dao/KubernetesClusterDao.java | 4 +-
.../cluster/dao/KubernetesClusterDaoImpl.java | 33 ++-
.../java/com/cloud/user/AccountManagerImpl.java | 15 ++
9 files changed, 307 insertions(+), 49 deletions(-)
diff --git
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
index a39c26680fd..924dbf37f67 100644
---
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
+++
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
@@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster;
import org.apache.cloudstack.acl.ControlledEntity;
+import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.Adapter;
@@ -25,4 +26,5 @@ public interface KubernetesServiceHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
void checkVmCanBeDestroyed(UserVm userVm);
+ void cleanupForAccount(Account account);
}
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
index 17b07496731..dae5f3a3467 100644
--- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
+++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
@@ -16,6 +16,9 @@
// under the License.
package com.cloud.user.dao;
+import java.util.Date;
+import java.util.List;
+
import com.cloud.user.Account;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
@@ -23,9 +26,6 @@ import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDao;
-import java.util.Date;
-import java.util.List;
-
public interface AccountDao extends GenericDao<AccountVO, Long> {
Pair<User, Account> findUserAccountByApiKey(String apiKey);
@@ -33,6 +33,8 @@ public interface AccountDao extends GenericDao<AccountVO,
Long> {
Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filter
filter);
+ List<AccountVO> findAccountsByName(String accountName);
+
List<AccountVO> findActiveAccounts(Long maxAccountId, Filter filter);
List<AccountVO> findRecentlyDeletedAccounts(Long maxAccountId, Date
earliestRemovedDate, Filter filter);
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
index 3dacbb70f39..be5ac2e8204 100644
--- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
@@ -16,6 +16,15 @@
// under the License.
package com.cloud.user.dao;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
import com.cloud.user.Account;
import com.cloud.user.Account.State;
import com.cloud.user.AccountVO;
@@ -30,15 +39,7 @@ import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
-import org.apache.commons.lang3.StringUtils;
import com.cloud.utils.db.TransactionLegacy;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Date;
-import java.util.List;
@Component
public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements
AccountDao {
@@ -180,6 +181,16 @@ public class AccountDaoImpl extends
GenericDaoBase<AccountVO, Long> implements A
return searchAndCount(sc, filter);
}
+ @Override
+ public List<AccountVO> findAccountsByName(String accountName) {
+ SearchBuilder<AccountVO> sb = createSearchBuilder();
+ sb.and("accountName", sb.entity().getAccountName(),
SearchCriteria.Op.EQ);
+ sb.done();
+ SearchCriteria<AccountVO> sc = sb.create();
+ sc.setParameters("accountName", accountName);
+ return search(sc, null);
+ }
+
@Override
public Account findEnabledAccount(String accountName, Long domainId) {
SearchCriteria<AccountVO> sc = AllFieldsSearch.create("accountName",
accountName);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
index 9c402f83b03..3fea585cbfe 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -41,6 +41,11 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.acl.Role;
+import org.apache.cloudstack.acl.RolePermissionEntity;
+import org.apache.cloudstack.acl.RoleService;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.acl.Rule;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
@@ -49,6 +54,14 @@ import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd;
+import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.DeleteFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.ListFirewallRulesCmd;
+import org.apache.cloudstack.api.command.user.firewall.UpdateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -59,6 +72,18 @@ import
org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetes
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRuleInstancesCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRulesCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.RemoveFromLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.UpdateLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworksCmd;
+import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.ListResponse;
@@ -145,6 +170,8 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.projects.Project;
+import com.cloud.projects.ProjectAccount;
+import com.cloud.projects.ProjectManager;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
@@ -153,15 +180,18 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
+import com.cloud.user.AccountVO;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.User;
import com.cloud.user.UserAccount;
import com.cloud.user.UserVO;
+import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.user.dao.UserDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
+import com.cloud.utils.UuidUtils;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -186,6 +216,33 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
private static final Logger LOGGER =
Logger.getLogger(KubernetesClusterManagerImpl.class);
private static final String
DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME =
"DefaultNetworkOfferingforKubernetesService";
+ private static final List<Class<?>>
PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList(
+ QueryAsyncJobResultCmd.class,
+ ListVMsCmd.class,
+ ListNetworksCmd.class,
+ ListPublicIpAddressesCmd.class,
+ AssociateIPAddrCmd.class,
+ DisassociateIPAddrCmd.class,
+ ListLoadBalancerRulesCmd.class,
+ CreateLoadBalancerRuleCmd.class,
+ UpdateLoadBalancerRuleCmd.class,
+ DeleteLoadBalancerRuleCmd.class,
+ AssignToLoadBalancerRuleCmd.class,
+ RemoveFromLoadBalancerRuleCmd.class,
+ ListLoadBalancerRuleInstancesCmd.class,
+ ListFirewallRulesCmd.class,
+ CreateFirewallRuleCmd.class,
+ UpdateFirewallRuleCmd.class,
+ DeleteFirewallRuleCmd.class,
+ ListNetworkACLsCmd.class,
+ CreateNetworkACLCmd.class,
+ DeleteNetworkACLCmd.class,
+ ListKubernetesClustersCmd.class,
+ ScaleKubernetesClusterCmd.class
+ );
+ private static final String PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME =
"Kubernetes";
+ private static final String PROJECT_KUBERNETES_ACCOUNT_LAST_NAME =
"Service User";
+
protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event,
KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
@@ -258,9 +315,14 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
public SecurityGroupService securityGroupService;
@Inject
public NetworkHelper networkHelper;
-
@Inject
private UserVmService userVmService;
+ @Inject
+ public AccountDao accountDao;
+ @Inject
+ public ProjectManager projectManager;
+ @Inject
+ RoleService roleService;
private void logMessage(final Level logLevel, final String message, final
Exception e) {
if (logLevel == Level.WARN) {
@@ -1371,23 +1433,31 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
}
- private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster)
{
- Account owner =
accountService.getActiveAccountById(kubernetesCluster.getAccountId());
- if (owner == null || owner.getType() == Account.Type.PROJECT) {
- owner = CallContext.current().getCallingAccount();
+ protected String[] createUserApiKeyAndSecretKey(long userId) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ return accountService.createApiKeyAndSecretKey(userId);
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected String[] getServiceUserKeys(Account owner) {
+ String username = owner.getAccountName();
+ if (!username.startsWith(KUBEADMIN_ACCOUNT_NAME + "-")) {
+ username += "-" + KUBEADMIN_ACCOUNT_NAME;
}
- String username = owner.getAccountName() + "-" +
KUBEADMIN_ACCOUNT_NAME;
UserAccount kubeadmin = accountService.getActiveUserAccount(username,
owner.getDomainId());
- String[] keys = null;
+ String[] keys;
if (kubeadmin == null) {
User kube = userDao.persist(new UserVO(owner.getAccountId(),
username, UUID.randomUUID().toString(), owner.getAccountName(),
- KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null,
UUID.randomUUID().toString(), User.Source.UNKNOWN));
- keys = accountService.createApiKeyAndSecretKey(kube.getId());
+ KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null,
UUID.randomUUID().toString(), User.Source.UNKNOWN));
+ keys = createUserApiKeyAndSecretKey(kube.getId());
} else {
String apiKey = kubeadmin.getApiKey();
String secretKey = kubeadmin.getSecretKey();
if (StringUtils.isAnyEmpty(apiKey, secretKey)) {
- keys =
accountService.createApiKeyAndSecretKey(kubeadmin.getId());
+ keys = createUserApiKeyAndSecretKey(kubeadmin.getId());
} else {
keys = new String[]{apiKey, secretKey};
}
@@ -1395,6 +1465,76 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
return keys;
}
+ protected Role createProjectKubernetesAccountRole() {
+ Role role =
roleService.createRole(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, RoleType.User,
+ PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, false);
+ for (Class<?> allowedApi :
PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS) {
+ final String apiName = BaseCmd.getCommandNameByClass(allowedApi);
+ roleService.createRolePermission(role, new Rule(apiName),
RolePermissionEntity.Permission.ALLOW,
+ String.format("Allow %s", apiName));
+ }
+ roleService.createRolePermission(role, new Rule("*"),
RolePermissionEntity.Permission.DENY,
+ "Deny all");
+ LOGGER.debug(String.format("Created default role for Kubernetes
service account in projects: %s", role));
+ return role;
+ }
+
+ public Role getProjectKubernetesAccountRole() {
+ List<Role> roles =
roleService.findRolesByName(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME);
+ if (CollectionUtils.isNotEmpty(roles)) {
+ Role role = roles.get(0);
+ LOGGER.debug(String.format("Found default role for Kubernetes
service account in projects: %s", role));
+ return role;
+ }
+ return createProjectKubernetesAccountRole();
+ }
+
+ protected Account createProjectKubernetesAccount(final Project project,
final String accountName) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ Role role = getProjectKubernetesAccountRole();
+ UserAccount userAccount =
accountService.createUserAccount(accountName,
+ UuidUtils.first(UUID.randomUUID().toString()),
PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME,
+ PROJECT_KUBERNETES_ACCOUNT_LAST_NAME, null, null,
accountName, Account.Type.NORMAL, role.getId(),
+ project.getDomainId(), null, null, null, null,
User.Source.NATIVE);
+ projectManager.assignAccountToProject(project,
userAccount.getAccountId(), ProjectAccount.Role.Regular,
+ userAccount.getId(), null);
+ Account account =
accountService.getAccount(userAccount.getAccountId());
+ LOGGER.debug(String.format("Created Kubernetes service account in
project %s: %s", project, account));
+ return account;
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected Account getProjectKubernetesAccount(final Account callerAccount,
final boolean create) {
+ Project project =
ApiDBUtils.findProjectByProjectAccountId(callerAccount.getId());
+ final String accountName = String.format("%s-%s",
KUBEADMIN_ACCOUNT_NAME, UuidUtils.first(project.getUuid()));
+ List<AccountVO> accounts = accountDao.findAccountsByName(accountName);
+ for (AccountVO account : accounts) {
+ if (projectManager.canAccessProjectAccount(account,
project.getProjectAccountId())) {
+ LOGGER.debug(String.format("Created Kubernetes service account
in project %s: %s", project, account));
+ return account;
+ }
+ }
+ return create ? createProjectKubernetesAccount(project, accountName) :
null;
+ }
+
+ protected Account getProjectKubernetesAccount(final Account callerAccount)
{
+ return getProjectKubernetesAccount(callerAccount, true);
+ }
+
+ private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster)
{
+ Account owner =
accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+ if (owner == null) {
+ owner = CallContext.current().getCallingAccount();
+ }
+ if (owner.getType() == Account.Type.PROJECT) {
+ owner = getProjectKubernetesAccount(owner);
+ }
+ return getServiceUserKeys(owner);
+ }
+
@Override
@ActionEvent(eventType =
KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP,
eventDescription = "stopping Kubernetes cluster", async = true)
@@ -1439,15 +1579,13 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
Long kubernetesClusterId = cmd.getId();
- KubernetesClusterVO cluster =
kubernetesClusterDao.findById(kubernetesClusterId);
+ final KubernetesClusterVO cluster =
kubernetesClusterDao.findById(kubernetesClusterId);
if (cluster == null) {
throw new InvalidParameterValueException("Invalid cluster id
specified");
}
accountManager.checkAccess(CallContext.current().getCallingAccount(),
SecurityChecker.AccessType.OperateEntry, false, cluster);
if (cluster.getClusterType() ==
KubernetesCluster.ClusterType.CloudManaged) {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(cluster, this);
- destroyWorker = ComponentContext.inject(destroyWorker);
- return destroyWorker.destroy();
+ return destroyKubernetesCluster(cluster);
} else {
boolean cleanup = cmd.getCleanup();
boolean expunge = cmd.getExpunge();
@@ -1470,13 +1608,14 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
}
}
- return Transaction.execute(new TransactionCallback<Boolean>() {
- @Override
- public Boolean doInTransaction(TransactionStatus status) {
-
kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
-
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
- return kubernetesClusterDao.remove(kubernetesClusterId);
+ return Transaction.execute((TransactionCallback<Boolean>) status
-> {
+ kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
+
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
+ if (kubernetesClusterDao.remove(kubernetesClusterId)) {
+ deleteProjectKubernetesAccountIfNeeded(cluster);
+ return true;
}
+ return false;
});
}
}
@@ -1730,6 +1869,66 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
return responseList;
}
+ protected void deleteProjectKubernetesAccount(Account projectAccount) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ Account serviceAccount =
getProjectKubernetesAccount(projectAccount, false);
+ if (serviceAccount != null) {
+
accountManager.deleteAccount(accountDao.findById(serviceAccount.getId()),
User.UID_SYSTEM,
+ accountService.getSystemAccount());
+ }
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected void deleteProjectKubernetesAccountIfNeeded(final
KubernetesCluster kubernetesCluster) {
+ Account owner =
accountService.getAccount(kubernetesCluster.getAccountId());
+ if (owner == null) {
+ return;
+ }
+ if (Account.Type.PROJECT.equals(owner.getType()) &&
+
kubernetesClusterDao.countNotForGCByAccount(owner.getAccountId()) == 0) {
+ deleteProjectKubernetesAccount(owner);
+ }
+ }
+
+ protected boolean destroyKubernetesCluster(KubernetesCluster
kubernetesCluster, boolean deleteProjectAccount) {
+ KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
+ KubernetesClusterManagerImpl.this);
+ destroyWorker = ComponentContext.inject(destroyWorker);
+ boolean result = destroyWorker.destroy();
+ if (deleteProjectAccount) {
+ deleteProjectKubernetesAccountIfNeeded(kubernetesCluster);
+ }
+ return result;
+ }
+
+ protected boolean destroyKubernetesCluster(KubernetesCluster
kubernetesCluster) {
+ return destroyKubernetesCluster(kubernetesCluster, true);
+ }
+
+ @Override
+ public void cleanupForAccount(Account account) {
+ List<KubernetesClusterVO> clusters =
kubernetesClusterDao.listForCleanupByAccount(account.getId());
+ if (CollectionUtils.isEmpty(clusters)) {
+ return;
+ }
+ LOGGER.debug(String.format("Cleaning up %d Kubernetes cluster for %s",
clusters.size(), account));
+ for (KubernetesClusterVO cluster : clusters) {
+ try {
+ destroyKubernetesCluster(cluster, false);
+ } catch (CloudRuntimeException e) {
+ LOGGER.warn(String.format("Failed to destroy Kubernetes
cluster: %s during cleanup for %s",
+ cluster.getName(), account), e);
+ }
+ }
+ if (!Account.Type.PROJECT.equals(account.getType())) {
+ return;
+ }
+ deleteProjectKubernetesAccount(account);
+ }
+
@Override
public List<Class<?>> getCommands() {
List<Class<?>> cmdList = new ArrayList<Class<?>>();
@@ -1781,9 +1980,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
LOGGER.info(String.format("Running Kubernetes cluster
garbage collector on Kubernetes cluster : %s", kubernetesCluster.getName()));
}
try {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
KubernetesClusterManagerImpl.this);
- destroyWorker = ComponentContext.inject(destroyWorker);
- if (destroyWorker.destroy()) {
+ if (destroyKubernetesCluster(kubernetesCluster)) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(String.format("Garbage collection
complete for Kubernetes cluster : %s", kubernetesCluster.getName()));
}
@@ -1909,9 +2106,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
LOGGER.info(String.format("Running Kubernetes
cluster state scanner on Kubernetes cluster : %s for state: %s",
kubernetesCluster.getName(), KubernetesCluster.State.Destroying.toString()));
}
try {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
KubernetesClusterManagerImpl.this);
- destroyWorker =
ComponentContext.inject(destroyWorker);
- destroyWorker.destroy();
+ destroyKubernetesCluster(kubernetesCluster);
} catch (Exception e) {
LOGGER.warn(String.format("Failed to run
Kubernetes cluster Destroying state scanner on Kubernetes cluster : %s status
scanner", kubernetesCluster.getName()), e);
}
@@ -1925,7 +2120,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
// checks if Kubernetes cluster is in desired state
- boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster,
VirtualMachine.State state) {
+ private boolean isClusterVMsInDesiredState(KubernetesCluster
kubernetesCluster, VirtualMachine.State state) {
List<KubernetesClusterVmMapVO> clusterVMs =
kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
// check cluster is running at desired capacity include control nodes
as well
@@ -1985,6 +2180,8 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
LOGGER.trace("Added service for the network offering: " +
offService);
}
+ getProjectKubernetesAccountRole();
+
_gcExecutor.scheduleWithFixedDelay(new
KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS);
_stateScanner.scheduleWithFixedDelay(new
KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
index 6acc876493e..20550496a6d 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
@@ -16,6 +16,8 @@
// under the License.
package com.cloud.kubernetes.cluster;
+import java.util.List;
+
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -33,16 +35,16 @@ import
org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClu
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
+import com.cloud.user.Account;
import com.cloud.utils.component.PluggableService;
import com.cloud.utils.exception.CloudRuntimeException;
-import java.util.List;
-
public interface KubernetesClusterService extends PluggableService,
Configurable {
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
+ String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service
Role";
static final ConfigKey<Boolean> KubernetesServiceEnabled = new
ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.service.enabled",
@@ -122,4 +124,6 @@ public interface KubernetesClusterService extends
PluggableService, Configurable
boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd);
List<RemoveVirtualMachinesFromKubernetesClusterResponse>
removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
+
+ void cleanupForAccount(Account account);
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
index 9551142d24e..57c0ede3654 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
@@ -34,6 +34,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
import com.cloud.kubernetes.version.KubernetesVersionEventTypes;
+import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.exception.CloudRuntimeException;
@@ -47,6 +48,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase
implements Kubernet
private KubernetesClusterDao kubernetesClusterDao;
@Inject
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
+ @Inject
+ KubernetesClusterService kubernetesClusterService;
protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass,
Class<?> entityClass) {
Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields();
@@ -94,6 +97,11 @@ public class KubernetesServiceHelperImpl extends AdapterBase
implements Kubernet
throw new CloudRuntimeException(msg);
}
+ @Override
+ public void cleanupForAccount(Account account) {
+ kubernetesClusterService.cleanupForAccount(account);
+ }
+
@Override
public String getConfigComponentName() {
return KubernetesServiceHelper.class.getSimpleName();
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
index 9341912012f..7df6a6b1dce 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
@@ -25,8 +25,8 @@ import com.cloud.utils.fsm.StateDao;
public interface KubernetesClusterDao extends GenericDao<KubernetesClusterVO,
Long>,
StateDao<KubernetesCluster.State, KubernetesCluster.Event,
KubernetesCluster> {
-
- List<KubernetesClusterVO> listByAccount(long accountId);
+ List<KubernetesClusterVO> listForCleanupByAccount(long accountId);
+ int countNotForGCByAccount(long accountId);
List<KubernetesClusterVO> findKubernetesClustersToGarbageCollect();
List<KubernetesClusterVO>
findManagedKubernetesClustersInState(KubernetesCluster.State state);
List<KubernetesClusterVO> listByNetworkId(long networkId);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
index 63cca3563f7..7bec98d5d25 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
@@ -30,16 +30,25 @@ import com.cloud.utils.db.TransactionLegacy;
@Component
public class KubernetesClusterDaoImpl extends
GenericDaoBase<KubernetesClusterVO, Long> implements KubernetesClusterDao {
- private final SearchBuilder<KubernetesClusterVO> AccountIdSearch;
+ private final SearchBuilder<KubernetesClusterVO> CleanupAccountIdSearch;
+ private final SearchBuilder<KubernetesClusterVO> NotForGCByAccountIDCount;
private final SearchBuilder<KubernetesClusterVO> GarbageCollectedSearch;
private final SearchBuilder<KubernetesClusterVO> ManagedStateSearch;
private final SearchBuilder<KubernetesClusterVO> SameNetworkSearch;
private final SearchBuilder<KubernetesClusterVO> KubernetesVersionSearch;
public KubernetesClusterDaoImpl() {
- AccountIdSearch = createSearchBuilder();
- AccountIdSearch.and("account",
AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
- AccountIdSearch.done();
+ CleanupAccountIdSearch = createSearchBuilder();
+ CleanupAccountIdSearch.and("account",
CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+ CleanupAccountIdSearch.and("cluster_type",
CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ);
+ CleanupAccountIdSearch.done();
+
+ NotForGCByAccountIDCount = createSearchBuilder();
+ NotForGCByAccountIDCount.and("gc",
NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.and("account",
NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.and("cluster_type",
NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null);
+ NotForGCByAccountIDCount.done();
GarbageCollectedSearch = createSearchBuilder();
GarbageCollectedSearch.and("gc",
GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ);
@@ -62,10 +71,20 @@ public class KubernetesClusterDaoImpl extends
GenericDaoBase<KubernetesClusterVO
}
@Override
- public List<KubernetesClusterVO> listByAccount(long accountId) {
- SearchCriteria<KubernetesClusterVO> sc = AccountIdSearch.create();
+ public List<KubernetesClusterVO> listForCleanupByAccount(long accountId) {
+ SearchCriteria<KubernetesClusterVO> sc =
CleanupAccountIdSearch.create();
+ sc.setParameters("cluster_type",
KubernetesCluster.ClusterType.CloudManaged);
+ sc.setParameters("account", accountId);
+ return listBy(sc);
+ }
+
+ @Override
+ public int countNotForGCByAccount(long accountId) {
+ SearchCriteria<KubernetesClusterVO> sc =
NotForGCByAccountIDCount.create();
+ sc.setParameters("cluster_type",
KubernetesCluster.ClusterType.CloudManaged);
sc.setParameters("account", accountId);
- return listBy(sc, null);
+ sc.setParameters("gc", false);
+ return getCount(sc);
}
@Override
diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java
b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
index 5ecdb40b714..180f4e49521 100644
--- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java
+++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
@@ -85,6 +85,7 @@ import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.jetbrains.annotations.NotNull;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd;
@@ -117,6 +118,7 @@ import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
@@ -171,6 +173,7 @@ import com.cloud.utils.ConstantTimeComparator;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
+import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.Manager;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.component.PluggableService;
@@ -859,6 +862,16 @@ public class AccountManagerImpl extends ManagerBase
implements AccountManager, M
return cleanupAccount(account, callerUserId, caller);
}
+ protected void cleanupPluginsResourcesIfNeeded(Account account) {
+ try {
+ KubernetesServiceHelper kubernetesServiceHelper =
+
ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class);
+ kubernetesServiceHelper.cleanupForAccount(account);
+ } catch (NoSuchBeanDefinitionException ignored) {
+ s_logger.debug("No KubernetesServiceHelper bean found");
+ }
+ }
+
protected boolean cleanupAccount(AccountVO account, long callerUserId,
Account caller) {
long accountId = account.getId();
boolean accountCleanupNeeded = false;
@@ -940,6 +953,8 @@ public class AccountManagerImpl extends ManagerBase
implements AccountManager, M
}
}
+ cleanupPluginsResourcesIfNeeded(account);
+
// Destroy the account's VMs
List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
if (s_logger.isDebugEnabled()) {
