Repository: stratos Updated Branches: refs/heads/master bf721e83b -> af0f6edd8
Fixing kubernetes service proxy creation logic Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/af0f6edd Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/af0f6edd Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/af0f6edd Branch: refs/heads/master Commit: af0f6edd8ecd8ba5eeb8907f1a0d335a15333122 Parents: bf721e8 Author: Imesh Gunaratne <[email protected]> Authored: Mon Dec 22 16:36:05 2014 +0530 Committer: Imesh Gunaratne <[email protected]> Committed: Mon Dec 22 16:36:19 2014 +0530 ---------------------------------------------------------------------- ...inerClusterContextToKubernetesContainer.java | 7 +- .../cloud/controller/iaases/KubernetesIaas.java | 88 ++++++++++++-------- .../impl/CloudControllerServiceImpl.java | 7 +- .../impl/CloudControllerServiceUtil.java | 12 +-- .../controller/util/CloudControllerUtil.java | 6 +- 5 files changed, 71 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/af0f6edd/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/functions/ContainerClusterContextToKubernetesContainer.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/functions/ContainerClusterContextToKubernetesContainer.java b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/functions/ContainerClusterContextToKubernetesContainer.java index 274ab75..5665d59 100644 --- a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/functions/ContainerClusterContextToKubernetesContainer.java +++ b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/functions/ContainerClusterContextToKubernetesContainer.java @@ -84,7 +84,7 @@ public class ContainerClusterContextToKubernetesContainer implements Function<Me for (PortMapping portMapping : cartridge.getPortMappings()) { Port p = new Port(); - p.setName(p.getProtocol() + p.getContainerPort()); + p.setName(p.getProtocol() + "-" + p.getContainerPort()); // In kubernetes transport protocol always be 'tcp' p.setProtocol("tcp"); p.setContainerPort(Integer.parseInt(portMapping.getPort())); @@ -120,6 +120,11 @@ public class ContainerClusterContextToKubernetesContainer implements Function<Me addToEnvironmentVariables(environmentVariables, StratosConstants.KUBERNETES_CLUSTER_ID, kubernetesClusterId); + if(log.isDebugEnabled()) { + log.debug(String.format("Environment variables: [cluster-id] %s [member-id] %s [variables] %s", + memberContext.getClusterId(), memberContext.getMemberId(), environmentVariables.toString())); + } + EnvironmentVariable[] array = new EnvironmentVariable[environmentVariables.size()]; return environmentVariables.toArray(array); } http://git-wip-us.apache.org/repos/asf/stratos/blob/af0f6edd/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/iaases/KubernetesIaas.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/iaases/KubernetesIaas.java b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/iaases/KubernetesIaas.java index 7d495ce..2ae3a85 100644 --- a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/iaases/KubernetesIaas.java +++ b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/iaases/KubernetesIaas.java @@ -53,7 +53,7 @@ import java.util.concurrent.locks.Lock; public class KubernetesIaas extends Iaas { private static final Log log = LogFactory.getLog(KubernetesIaas.class); - private static final long POD_CREATION_TIMEOUT = 60000; // 1 min + private static final long POD_CREATION_TIMEOUT = 120000; // 1 min public static final String PAYLOAD_PARAMETER_SEPARATOR = ","; public static final String PAYLOAD_PARAMETER_NAME_VALUE_SEPARATOR = "="; @@ -163,31 +163,36 @@ public class KubernetesIaas extends Iaas { StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT); KubernetesClusterContext kubClusterContext = getKubernetesClusterContext(kubernetesClusterId, - kubernetesMasterIp, kubernetesMasterPort, kubernetesPortRange.getLower(), - kubernetesPortRange.getUpper()); + kubernetesMasterIp, kubernetesMasterPort, kubernetesPortRange.getUpper(), + kubernetesPortRange.getLower()); // Get kubernetes API KubernetesApiClient kubernetesApi = kubClusterContext.getKubApi(); - // Create replication controller - createReplicationController(memberContext, clusterId, kubernetesApi); - // Create proxy services for port mappings List<Service> services = createProxyServices(clusterContext, kubClusterContext, kubernetesApi); clusterContext.setKubernetesServices(services); CloudControllerContext.getInstance().updateClusterContext(clusterContext); + // Create replication controller + createReplicationController(memberContext, clusterId, kubernetesApi); + // Wait for pod to be created - Pod[] pods = waitForPodToBeCreated(memberContext, kubernetesApi); - if (pods.length != 1) { + List<Pod> pods = waitForPodToBeCreated(memberContext, kubernetesApi); + if (pods.size() != 1) { if (log.isDebugEnabled()) { log.debug(String.format("Pod did not create within %d sec, hence deleting the service: " + "[cluster-id] %s [member-id] %s", ((int)POD_CREATION_TIMEOUT/1000), clusterId, memberId)); } - terminateContainers(clusterId); - return null; + try { + terminateContainers(clusterId); + } catch (Exception e) { + String message = "Could not terminate containers which were partially created"; + log.error(message, e); + throw new RuntimeException(message, e); + } } - Pod pod = pods[0]; + Pod pod = pods.get(0); if (log.isDebugEnabled()) { log.debug(String.format("Pod created: [cluster-id] %s [member-id] %s [pod-id] %s", clusterId, memberId, pod.getId())); @@ -212,7 +217,8 @@ public class KubernetesIaas extends Iaas { return newMemberContext; } catch (Exception e) { - String msg = "Could not start container: " + memberContext.toString() + " Cause: " + e.getMessage(); + String msg = String.format("Could not start container: [cartridge-type] %s [member-id] %s", + memberContext.getCartridgeType(), memberContext.getMemberId()); log.error(msg, e); throw new IllegalStateException(msg, e); } @@ -241,22 +247,29 @@ public class KubernetesIaas extends Iaas { return newMemberContext; } - private Pod[] waitForPodToBeCreated(MemberContext memberContext, KubernetesApiClient kubernetesApi) throws KubernetesClientException, InterruptedException { + private List<Pod> waitForPodToBeCreated(MemberContext memberContext, KubernetesApiClient kubernetesApi) throws KubernetesClientException, InterruptedException { Labels labels = new Labels(); - labels.setName(memberContext.getMemberId()); - Pod[] pods = new Pod[0]; + labels.setName(memberContext.getClusterId()); + List<Pod> podList = new ArrayList<Pod>(); long startTime = System.currentTimeMillis(); - while (pods.length == 1) { - pods = kubernetesApi.queryPods(new Labels[]{labels}); + while (podList.size() == 0) { + Pod[] pods = kubernetesApi.queryPods(new Labels[]{labels}); + if((pods != null) && (pods.length > 0)){ + for(Pod pod : pods) { + if(pod != null) { + podList.add(pod); + } + } + } if (log.isDebugEnabled()) { - log.debug("Member pod: [member-id] " + memberContext.getMemberId() + " [count] " + pods.length); + log.debug("Member pod: [member-id] " + memberContext.getMemberId() + " [count] " + podList.size()); } if ((System.currentTimeMillis() - startTime) > POD_CREATION_TIMEOUT) { break; } Thread.sleep(5000); } - return pods; + return podList; } /** @@ -281,8 +294,8 @@ public class KubernetesIaas extends Iaas { // Create replication controller ReplicationController replicationController = controllerFunction.apply(memberContext); kubernetesApi.createReplicationController(replicationController); - if (log.isDebugEnabled()) { - log.debug("Kubernetes replication controller successfully created: [cluster-id] " + clusterId); + if (log.isInfoEnabled()) { + log.info("Kubernetes replication controller successfully created: [cluster-id] " + clusterId); } return replicationController; } @@ -311,22 +324,27 @@ public class KubernetesIaas extends Iaas { List<PortMapping> portMappings = cartridge.getPortMappings(); for(PortMapping portMapping : portMappings) { - if (log.isInfoEnabled()) { - log.info(String.format("Creating kubernetes service: [cluster-id] %s [protocol] %s [port] %s ", - clusterId, portMapping.getProtocol(), portMapping.getPort())); - } - + String serviceId = prepareKubernetesServiceId(clusterId, portMapping); int nextServicePort = kubernetesClusterContext.getNextServicePort(); if(nextServicePort == -1) { - throw new RuntimeException("Service port not found"); + throw new RuntimeException(String.format("Could not generate service port: [cluster-id] %s ", + clusterContext.getClusterId())); + } + + if (log.isInfoEnabled()) { + log.info(String.format("Creating kubernetes service: [cluster-id] %s [service-id] %s " + + "[protocol] %s [service-port] %d [container-port] %s [proxy-port] %s", clusterId, + serviceId, portMapping.getProtocol(), nextServicePort, portMapping.getPort(), + portMapping.getProxyPort())); } Service service = new Service(); - service.setId(prepareKubernetesServiceId(clusterId, portMapping)); + service.setId(serviceId); service.setApiVersion("v1beta1"); service.setKind("Service"); service.setPort(nextServicePort); service.setContainerPort(portMapping.getPort()); + Selector selector = new Selector(); selector.setName(clusterId); service.setSelector(selector); @@ -335,9 +353,10 @@ public class KubernetesIaas extends Iaas { services.add(service); if (log.isInfoEnabled()) { - log.info(String.format("Kubernetes service successfully created: [cluster-id] %s [protocol] %s " + - "[port] %s [proxy-port] %s", clusterId, portMapping.getProtocol(), - portMapping.getPort(), service.getPort())); + log.info(String.format("Kubernetes service successfully created: [cluster-id] %s [service-id] %s " + + "[protocol] %s [service-port] %d [container-port] %s [proxy-port] %s", clusterId, + service.getId(), portMapping.getProtocol(), service.getPort(), portMapping.getPort(), + portMapping.getProxyPort())); } } return services; @@ -345,12 +364,15 @@ public class KubernetesIaas extends Iaas { /** * Prepare kubernetes service id using clusterId, port protocol and port. - * @param clusterId * @param portMapping * @return */ private String prepareKubernetesServiceId(String clusterId, PortMapping portMapping) { - return String.format("%s-%s-%s", clusterId, portMapping.getProtocol(), portMapping.getPort()); + String serviceId = String.format("%s-%s-%s", clusterId, portMapping.getProtocol(), portMapping.getPort()); + if(serviceId.contains(".")) { + serviceId = serviceId.replace(".", "-"); + } + return serviceId; } /** http://git-wip-us.apache.org/repos/asf/stratos/blob/af0f6edd/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceImpl.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceImpl.java b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceImpl.java index 9500920..6eff63f 100644 --- a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceImpl.java +++ b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceImpl.java @@ -1094,7 +1094,8 @@ public class CloudControllerServiceImpl implements CloudControllerService { lock = CloudControllerContext.getInstance().acquireKubernetesClusterWriteLock(); if (log.isInfoEnabled()) { - log.info("Adding kubernetes cluster: " + kubernetesCluster); + log.info(String.format("Adding kubernetes cluster: [kubernetes-cluster-id] %s", + kubernetesCluster.getClusterId())); } CloudControllerUtil.validateKubernetesCluster(kubernetesCluster); @@ -1103,8 +1104,8 @@ public class CloudControllerServiceImpl implements CloudControllerService { CloudControllerContext.getInstance().persist(); if (log.isInfoEnabled()) { - log.info(String.format("Kubernetes cluster added successfully: [id] %s, [description] %s", - kubernetesCluster.getClusterId(), kubernetesCluster.getDescription())); + log.info(String.format("Kubernetes cluster added successfully: [kubernetes-cluster-id] %s", + kubernetesCluster.getClusterId())); } return true; } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/stratos/blob/af0f6edd/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceUtil.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceUtil.java b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceUtil.java index 23db8db..f86ec9d 100644 --- a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceUtil.java +++ b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/services/impl/CloudControllerServiceUtil.java @@ -90,20 +90,12 @@ public class CloudControllerServiceUtil { } public static IaasProvider validatePartitionAndGetIaasProvider(Partition partition, IaasProvider iaasProvider) throws InvalidPartitionException { - String provider = partition.getProvider(); - Properties partitionProperties = CloudControllerUtil.toJavaUtilProperties(partition.getProperties()); - - if (CloudControllerConstants.KUBERNETES_PARTITION_PROVIDER.equals(provider)) { - // if this is a kubernetes based Partition - KubernetesPartitionValidator validator = new KubernetesPartitionValidator(); - validator.validate(partition, partitionProperties); - return iaasProvider; - - } else if (iaasProvider != null) { + if (iaasProvider != null) { // if this is a IaaS based partition Iaas iaas = iaasProvider.getIaas(); PartitionValidator validator = iaas.getPartitionValidator(); validator.setIaasProvider(iaasProvider); + Properties partitionProperties = CloudControllerUtil.toJavaUtilProperties(partition.getProperties()); iaasProvider = validator.validate(partition, partitionProperties); return iaasProvider; http://git-wip-us.apache.org/repos/asf/stratos/blob/af0f6edd/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/util/CloudControllerUtil.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/util/CloudControllerUtil.java b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/util/CloudControllerUtil.java index d29d9d9..0bca0bd 100644 --- a/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/util/CloudControllerUtil.java +++ b/components/org.apache.stratos.cloud.controller/src/main/java/org/apache/stratos/cloud/controller/util/CloudControllerUtil.java @@ -392,8 +392,10 @@ public class CloudControllerUtil { kubernetesCluster.getPortRange().getLower() > CloudControllerConstants.PORT_RANGE_MAX || kubernetesCluster.getPortRange().getLower() < CloudControllerConstants.PORT_RANGE_MIN || kubernetesCluster.getPortRange().getUpper() < kubernetesCluster.getPortRange().getLower()) { - throw new InvalidKubernetesClusterException("Port range is invalid " + - "for the Kubernetes cluster [id]" + kubernetesCluster.getClusterId()); + throw new InvalidKubernetesClusterException("Port range is invalid in kubernetes cluster " + + "[kubenetes-cluster-id] " + kubernetesCluster.getClusterId() + " " + + " [valid-min] " + CloudControllerConstants.PORT_RANGE_MIN + " [valid-max] " + + CloudControllerConstants.PORT_RANGE_MAX); } try { validateKubernetesMaster(kubernetesCluster.getKubernetesMaster());
