This is an automated email from the ASF dual-hosted git repository. nicknezis pushed a commit to branch nicknezis/k8s-secrets in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 7604fcbe8a079c731856c32a6ddb05f7c2f4f82f Author: Nicholas Nezis <[email protected]> AuthorDate: Thu Jul 8 23:14:25 2021 -0400 Added support for Secrets and SecretKeyRefs --- .../scheduler/kubernetes/KubernetesContext.java | 71 +++++++++++++--------- .../heron/scheduler/kubernetes/V1Controller.java | 47 ++++++++++++++ .../heron/scheduler/kubernetes/VolumesTests.java | 16 ++--- 3 files changed, 97 insertions(+), 37 deletions(-) diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java index 4074fbc..71a660a 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java @@ -59,44 +59,49 @@ public final class KubernetesContext extends Context { public static final String KUBERNETES_RESOURCE_REQUEST_MODE = "heron.kubernetes.resource.request.mode"; - public static final String HERON_KUBERNETES_VOLUME_NAME = "heron.kubernetes.volume.name"; - public static final String HERON_KUBERNETES_VOLUME_TYPE = "heron.kubernetes.volume.type"; + public static final String KUBERNETES_VOLUME_NAME = "heron.kubernetes.volume.name"; + public static final String KUBERNETES_VOLUME_TYPE = "heron.kubernetes.volume.type"; // HostPath volume keys // https://kubernetes.io/docs/concepts/storage/volumes/#hostpath - public static final String HERON_KUBERNETES_VOLUME_HOSTPATH_PATH = + public static final String KUBERNETES_VOLUME_HOSTPATH_PATH = "heron.kubernetes.volume.hostPath.path"; // nfs volume keys // https://kubernetes.io/docs/concepts/storage/volumes/#nfs - public static final String HERON_KUBERNETES_VOLUME_NFS_PATH = + public static final String KUBERNETES_VOLUME_NFS_PATH = "heron.kubernetes.volume.nfs.path"; - public static final String HERON_KUBERNETES_VOLUME_NFS_SERVER = + public static final String KUBERNETES_VOLUME_NFS_SERVER = "heron.kubernetes.volume.nfs.server"; // awsElasticBlockStore volume keys // https://kubernetes.io/docs/concepts/storage/volumes/#awselasticblockstore - public static final String HERON_KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID = + public static final String KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID = "heron.kubernetes.volume.awsElasticBlockStore.volumeID"; - public static final String HERON_KUBERNETES_VOLUME_AWS_EBS_FS_TYPE = + public static final String KUBERNETES_VOLUME_AWS_EBS_FS_TYPE = "heron.kubernetes.volume.awsElasticBlockStore.fsType"; // container mount volume mount keys - public static final String HERON_KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME = + public static final String KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME = "heron.kubernetes.container.volumeMount.name"; - public static final String HERON_KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH = + public static final String KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH = "heron.kubernetes.container.volumeMount.path"; - public static final String HERON_KUBERNETES_POD_ANNOTATION = + public static final String KUBERNETES_POD_ANNOTATION_PREFIX = "heron.kubernetes.pod.annotation."; - public static final String HERON_KUBERNETES_SERVICE_ANNOTATION = + public static final String KUBERNETES_SERVICE_ANNOTATION_PREFIX = "heron.kubernetes.service.annotation."; - public static final String HERON_KUBERNETES_POD_LABEL = - "heron.kubernetes.pod.label."; - public static final String HERON_KUBERNETES_SERVICE_LABEL = - "heron.kubernetes.service.label."; - + public static final String KUBERNETES_POD_LABEL_PREFIX = + "heron.kubernetes.pod.label."; + public static final String KUBERNETES_SERVICE_LABEL_PREFIX = + "heron.kubernetes.service.label."; + // heron.kubernetes.pod.secret.heron-secret=/etc/secrets + public static final String KUBERNETES_POD_SECRET_PREFIX = + "heron.kubernetes.pod.secret."; + // heron.kubernetes.pod.secretKeyRef.ENV_NAME=name:key + public static final String KUBERNETES_POD_SECRET_KEY_REF_PREFIX = + "heron.kubernetes.pod.secretKeyRef."; private KubernetesContext() { } @@ -128,31 +133,31 @@ public final class KubernetesContext extends Context { } static String getVolumeType(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_TYPE); + return config.getStringValue(KUBERNETES_VOLUME_TYPE); } static String getVolumeName(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_NAME); + return config.getStringValue(KUBERNETES_VOLUME_NAME); } static String getHostPathVolumePath(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_HOSTPATH_PATH); + return config.getStringValue(KUBERNETES_VOLUME_HOSTPATH_PATH); } static String getNfsVolumePath(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_NFS_PATH); + return config.getStringValue(KUBERNETES_VOLUME_NFS_PATH); } static String getNfsServer(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_NFS_SERVER); + return config.getStringValue(KUBERNETES_VOLUME_NFS_SERVER); } static String getAwsEbsVolumeId(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID); + return config.getStringValue(KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID); } static String getAwsEbsFsType(Config config) { - return config.getStringValue(HERON_KUBERNETES_VOLUME_AWS_EBS_FS_TYPE); + return config.getStringValue(KUBERNETES_VOLUME_AWS_EBS_FS_TYPE); } static boolean hasVolume(Config config) { @@ -160,27 +165,35 @@ public final class KubernetesContext extends Context { } static String getContainerVolumeName(Config config) { - return config.getStringValue(HERON_KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME); + return config.getStringValue(KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME); } static String getContainerVolumeMountPath(Config config) { - return config.getStringValue(HERON_KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH); + return config.getStringValue(KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH); } public static Map<String, String> getPodLabels(Config config) { - return getConfigItemsByPrefix(config, HERON_KUBERNETES_POD_LABEL); + return getConfigItemsByPrefix(config, KUBERNETES_POD_LABEL_PREFIX); } public static Map<String, String> getServiceLabels(Config config) { - return getConfigItemsByPrefix(config, HERON_KUBERNETES_SERVICE_LABEL); + return getConfigItemsByPrefix(config, KUBERNETES_SERVICE_LABEL_PREFIX); } public static Map<String, String> getPodAnnotations(Config config) { - return getConfigItemsByPrefix(config, HERON_KUBERNETES_POD_ANNOTATION); + return getConfigItemsByPrefix(config, KUBERNETES_POD_ANNOTATION_PREFIX); } public static Map<String, String> getServiceAnnotations(Config config) { - return getConfigItemsByPrefix(config, HERON_KUBERNETES_SERVICE_ANNOTATION); + return getConfigItemsByPrefix(config, KUBERNETES_SERVICE_ANNOTATION_PREFIX); + } + + public static Map<String, String> getPodSecretsToMount(Config config) { + return getConfigItemsByPrefix(config, KUBERNETES_POD_SECRET_PREFIX); + } + + public static Map<String, String> getPodSecretKeyRefs(Config config) { + return getConfigItemsByPrefix(config, KUBERNETES_POD_SECRET_KEY_REF_PREFIX); } static Set<String> getConfigKeys(Config config, String keyPrefix) { diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java index 2bf815b..35564e7 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java @@ -52,6 +52,7 @@ import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1ContainerPort; import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1EnvVarBuilder; import io.kubernetes.client.openapi.models.V1EnvVarSource; import io.kubernetes.client.openapi.models.V1LabelSelector; import io.kubernetes.client.openapi.models.V1ObjectFieldSelector; @@ -59,6 +60,7 @@ import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1PodSpec; import io.kubernetes.client.openapi.models.V1PodTemplateSpec; import io.kubernetes.client.openapi.models.V1ResourceRequirements; +import io.kubernetes.client.openapi.models.V1SecretVolumeSourceBuilder; import io.kubernetes.client.openapi.models.V1Service; import io.kubernetes.client.openapi.models.V1ServiceSpec; import io.kubernetes.client.openapi.models.V1StatefulSet; @@ -444,6 +446,8 @@ public class V1Controller extends KubernetesController { addVolumesIfPresent(podSpec); + mountSecretsAsVolumes(podSpec); + return podSpec; } @@ -473,6 +477,25 @@ public class V1Controller extends KubernetesController { } } + private void mountSecretsAsVolumes(V1PodSpec podSpec) { + final Config config = getConfiguration(); + final Map<String, String> secrets = KubernetesContext.getPodSecretsToMount(config); + for (Map.Entry<String, String> secret : secrets.entrySet()) { + final V1VolumeMount mount = new V1VolumeMount() + .name(secret.getKey()) + .mountPath(secret.getValue()); + final V1Volume secretVolume = new V1Volume() + .name(secret.getKey()) + .secret(new V1SecretVolumeSourceBuilder() + .withSecretName(secret.getKey()) + .build()); + podSpec.addVolumesItem(secretVolume); + for (V1Container container : podSpec.getContainers()) { + container.addVolumeMountsItem(mount); + } + } + } + private V1Container getContainer(List<String> executorCommand, Resource resource, int numberOfInstances) { final Config configuration = getConfiguration(); @@ -502,6 +525,7 @@ public class V1Controller extends KubernetesController { .fieldPath(KubernetesConstants.POD_NAME))); container.setEnv(Arrays.asList(envVarHost, envVarPodName)); + setSecretKeyRefs(container); // set container resources final V1ResourceRequirements resourceRequirements = new V1ResourceRequirements(); @@ -573,6 +597,29 @@ public class V1Controller extends KubernetesController { } } + private void setSecretKeyRefs(V1Container container) { + final Config config = getConfiguration(); + final Map<String, String> podSecretKeyRefs = KubernetesContext.getPodSecretKeyRefs(config); + for (Map.Entry<String, String> secret : podSecretKeyRefs.entrySet()) { + final String[] keyRefParts = secret.getValue().split(":"); + if (keyRefParts.length != 2) { + LOG.log(Level.SEVERE, "SecretKeyRef must be in the form name:key. <" + keyRefParts + ">"); + throw new TopologyRuntimeManagementException("SecretKeyRef must be in the form name:key. <" + keyRefParts + ">"); + } + String name = keyRefParts[0]; + String key = keyRefParts[1]; + V1EnvVar envVar = new V1EnvVarBuilder() + .withName(secret.getKey()) + .withNewValueFrom() + .withNewSecretKeyRef() + .withKey(key) + .withName(name) + .endSecretKeyRef() + .endValueFrom().build(); + container.addEnvItem(envVar); + } + } + public static double roundDecimal(double value, int places) { double scale = Math.pow(10, places); return Math.round(value * scale) / scale; diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java index 95f82f4..401c97f 100644 --- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java +++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/VolumesTests.java @@ -39,8 +39,8 @@ public class VolumesTests { public void testHostPathVolume() { final String path = "/test/dir1"; final Config config = Config.newBuilder() - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_TYPE, "hostPath") - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_HOSTPATH_PATH, path) + .put(KubernetesContext.KUBERNETES_VOLUME_TYPE, "hostPath") + .put(KubernetesContext.KUBERNETES_VOLUME_HOSTPATH_PATH, path) .build(); final V1Volume volume = Volumes.get().create(config); @@ -54,9 +54,9 @@ public class VolumesTests { final String path = "/test/dir1"; final String server = "10.10.10.10"; final Config config = Config.newBuilder() - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_TYPE, "nfs") - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_NFS_PATH, path) - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_NFS_SERVER, server) + .put(KubernetesContext.KUBERNETES_VOLUME_TYPE, "nfs") + .put(KubernetesContext.KUBERNETES_VOLUME_NFS_PATH, path) + .put(KubernetesContext.KUBERNETES_VOLUME_NFS_SERVER, server) .build(); final V1Volume volume = Volumes.get().create(config); @@ -71,9 +71,9 @@ public class VolumesTests { final String volumeId = "aws-ebs-1"; final String fsType = "ext4"; final Config config = Config.newBuilder() - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_TYPE, "awsElasticBlockStore") - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID, volumeId) - .put(KubernetesContext.HERON_KUBERNETES_VOLUME_AWS_EBS_FS_TYPE, fsType) + .put(KubernetesContext.KUBERNETES_VOLUME_TYPE, "awsElasticBlockStore") + .put(KubernetesContext.KUBERNETES_VOLUME_AWS_EBS_VOLUME_ID, volumeId) + .put(KubernetesContext.KUBERNETES_VOLUME_AWS_EBS_FS_TYPE, fsType) .build(); final V1Volume volume = Volumes.get().create(config);
