This is an automated email from the ASF dual-hosted git repository.
saadurrahman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new c6631cc [Heron-3724] Separate the Manager and Executors. (#3741)
c6631cc is described below
commit c6631ccfc8ec2f006ba8e14e0657778e79265dbe
Author: Saad Ur Rahman <[email protected]>
AuthorDate: Sun Dec 5 12:10:39 2021 -0500
[Heron-3724] Separate the Manager and Executors. (#3741)
All of the changes below are pertaining to the Kubernetes Scheduler.
* Topology's Executors and Manager separated into individual StatefulSets.
* Added the ability to specify individual Pod Templates for a topology's
Executors and Manager via the CLI.
* Added the ability to specify Persistent Volume Claims for a topology's
Executors and Manager via the CLI.
* Added the ability to specify resource Requests and Limits for a
topology's Executors and Manager via the CLI.
---
deploy/kubernetes/general/apiserver.yaml | 2 +-
deploy/kubernetes/helm/templates/tools.yaml | 3 +-
deploy/kubernetes/minikube/apiserver.yaml | 2 +-
.../scheduler/kubernetes/KubernetesConstants.java | 4 +
.../scheduler/kubernetes/KubernetesContext.java | 51 +-
.../heron/scheduler/kubernetes/V1Controller.java | 360 ++++++++++----
.../kubernetes/KubernetesContextTest.java | 169 ++++---
.../scheduler/kubernetes/V1ControllerTest.java | 424 ++++++++++++-----
.../docs/schedulers-k8s-execution-environment.md | 527 +++++++++++++++++++++
.../schedulers-k8s-persistent-volume-claims.md | 257 ----------
website2/docs/schedulers-k8s-pod-templates.md | 201 --------
website2/website/sidebars.json | 3 +-
12 files changed, 1256 insertions(+), 747 deletions(-)
diff --git a/deploy/kubernetes/general/apiserver.yaml
b/deploy/kubernetes/general/apiserver.yaml
index 094715a..2e6290c 100644
--- a/deploy/kubernetes/general/apiserver.yaml
+++ b/deploy/kubernetes/general/apiserver.yaml
@@ -91,7 +91,7 @@ spec:
-D
heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heron
-D
heron.statefulstorage.classname=org.apache.heron.statefulstorage.dlog.DlogStorage
-D
heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heron
- -D heron.kubernetes.pod.template.configmap.disabled=false
+ -D heron.kubernetes.pod.template.disabled=false
-D heron.kubernetes.persistent.volume.claims.cli.disabled=false
---
diff --git a/deploy/kubernetes/helm/templates/tools.yaml
b/deploy/kubernetes/helm/templates/tools.yaml
index c776e49..f418779d6 100644
--- a/deploy/kubernetes/helm/templates/tools.yaml
+++ b/deploy/kubernetes/helm/templates/tools.yaml
@@ -158,7 +158,7 @@ spec:
-D
heron.class.repacking.algorithm=org.apache.heron.packing.binpacking.FirstFitDecreasingPacking
{{- end }}
-D heron.kubernetes.resource.request.mode={{
.Values.topologyResourceRequestMode }}
- -D heron.kubernetes.pod.template.configmap.disabled={{
.Values.disablePodTemplates }}
+ -D heron.kubernetes.pod.template.disabled={{
.Values.disablePodTemplates }}
-D heron.kubernetes.persistent.volume.claims.cli.disabled={{
.Values.disablePersistentVolumeMountsCLI }}
envFrom:
- configMapRef:
@@ -297,6 +297,7 @@ rules:
- patch
- update
- watch
+ - deletecollection
- apiGroups:
- ""
resources:
diff --git a/deploy/kubernetes/minikube/apiserver.yaml
b/deploy/kubernetes/minikube/apiserver.yaml
index 53d879a..ce20c90 100644
--- a/deploy/kubernetes/minikube/apiserver.yaml
+++ b/deploy/kubernetes/minikube/apiserver.yaml
@@ -82,7 +82,7 @@ spec:
-D
heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heronbkdl
-D
heron.statefulstorage.classname=org.apache.heron.statefulstorage.dlog.DlogStorage
-D
heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heronbkdl
- -D heron.kubernetes.pod.template.configmap.disabled=false
+ -D heron.kubernetes.pod.template.disabled=false
-D heron.kubernetes.persistent.volume.claims.cli.disabled=false
---
diff --git
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesConstants.java
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesConstants.java
index 023d8bc..c89a0c6 100644
---
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesConstants.java
+++
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesConstants.java
@@ -36,6 +36,7 @@ public final class KubernetesConstants {
public static final String CPU = "cpu";
public static final String EXECUTOR_NAME = "executor";
+ public static final String MANAGER_NAME = "manager";
// container env constants
public static final String ENV_HOST = "HOST";
@@ -89,6 +90,9 @@ public final class KubernetesConstants {
public static final String JOB_LINK =
"/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy/#/pod";
+ protected static final String JSON_PATCH_STATEFUL_SET_REPLICAS_FORMAT =
+ "[{\"op\":\"replace\",\"path\":\"/spec/replicas\",\"value\":%d}]";
+
public static final Pattern VALID_POD_NAME_REGEX =
Pattern.compile("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*",
Pattern.CASE_INSENSITIVE);
diff --git
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java
index 40b7618..83aa0b2 100644
---
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java
+++
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java
@@ -86,11 +86,11 @@ public final class KubernetesContext extends Context {
public static final String KUBERNETES_VOLUME_AWS_EBS_FS_TYPE =
"heron.kubernetes.volume.awsElasticBlockStore.fsType";
- // pod template configmap
- public static final String KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME =
- "heron.kubernetes.pod.template.configmap.name";
- public static final String KUBERNETES_POD_TEMPLATE_CONFIGMAP_DISABLED =
- "heron.kubernetes.pod.template.configmap.disabled";
+ // Pod Template ConfigMap: heron.kubernetes.[executor | manager].pod.template
+ public static final String KUBERNETES_POD_TEMPLATE_LOCATION =
+ "heron.kubernetes.%s.pod.template";
+ public static final String KUBERNETES_POD_TEMPLATE_DISABLED =
+ "heron.kubernetes.pod.template.disabled";
// container mount volume mount keys
public static final String KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME =
@@ -116,9 +116,15 @@ public final class KubernetesContext extends Context {
// Persistent Volume Claims
public static final String KUBERNETES_PERSISTENT_VOLUME_CLAIMS_CLI_DISABLED =
"heron.kubernetes.persistent.volume.claims.cli.disabled";
- //
heron.kubernetes.volumes.persistentVolumeClaim.VOLUME_NAME.OPTION=OPTION_VALUE
+ // heron.kubernetes.[executor |
manager].volumes.persistentVolumeClaim.VOLUME_NAME.OPTION=VALUE
public static final String KUBERNETES_VOLUME_CLAIM_PREFIX =
- "heron.kubernetes.volumes.persistentVolumeClaim.";
+ "heron.kubernetes.%s.volumes.persistentVolumeClaim.";
+ // heron.kubernetes.[executor | manager].limits.OPTION=VALUE
+ public static final String KUBERNETES_RESOURCE_LIMITS_PREFIX =
+ "heron.kubernetes.%s.limits.";
+ // heron.kubernetes.[executor | manager].requests.OPTION=VALUE
+ public static final String KUBERNETES_RESOURCE_REQUESTS_PREFIX =
+ "heron.kubernetes.%s.requests.";
private KubernetesContext() {
}
@@ -189,12 +195,14 @@ public final class KubernetesContext extends Context {
return config.getStringValue(KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH);
}
- public static String getPodTemplateConfigMapName(Config config) {
- return config.getStringValue(KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME);
+ public static String getPodTemplateConfigMapName(Config config, boolean
isExecutor) {
+ final String key = String.format(KUBERNETES_POD_TEMPLATE_LOCATION,
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME :
KubernetesConstants.MANAGER_NAME);
+ return config.getStringValue(key);
}
- public static boolean getPodTemplateConfigMapDisabled(Config config) {
- final String disabled =
config.getStringValue(KUBERNETES_POD_TEMPLATE_CONFIGMAP_DISABLED);
+ public static boolean getPodTemplateDisabled(Config config) {
+ final String disabled =
config.getStringValue(KUBERNETES_POD_TEMPLATE_DISABLED);
return "true".equalsIgnoreCase(disabled);
}
@@ -222,6 +230,18 @@ public final class KubernetesContext extends Context {
return getConfigItemsByPrefix(config,
KUBERNETES_POD_SECRET_KEY_REF_PREFIX);
}
+ public static Map<String, String> getResourceLimits(Config config, boolean
isExecutor) {
+ final String key = String.format(KUBERNETES_RESOURCE_LIMITS_PREFIX,
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME :
KubernetesConstants.MANAGER_NAME);
+ return getConfigItemsByPrefix(config, key);
+ }
+
+ public static Map<String, String> getResourceRequests(Config config, boolean
isExecutor) {
+ final String key = String.format(KUBERNETES_RESOURCE_REQUESTS_PREFIX,
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME :
KubernetesConstants.MANAGER_NAME);
+ return getConfigItemsByPrefix(config, key);
+ }
+
public static boolean getPersistentVolumeClaimDisabled(Config config) {
final String disabled =
config.getStringValue(KUBERNETES_PERSISTENT_VOLUME_CLAIMS_CLI_DISABLED);
return "true".equalsIgnoreCase(disabled);
@@ -231,15 +251,18 @@ public final class KubernetesContext extends Context {
* Collects parameters form the <code>CLI</code> and generates a mapping
between <code>Volumes</code>
* and their configuration <code>key-value</code> pairs.
* @param config Contains the configuration options collected from the
<code>CLI</code>.
+ * @param isExecutor Flag used to collect CLI commands for the
<code>executor</code> and <code>manager</code>.
* @return A mapping between <code>Volumes</code> and their configuration
<code>key-value</code> pairs.
* Will return an empty list if there are no Volume Claim Templates to be
generated.
*/
public static Map<String,
Map<KubernetesConstants.VolumeClaimTemplateConfigKeys, String>>
- getVolumeClaimTemplates(Config config) {
+ getVolumeClaimTemplates(Config config, boolean isExecutor) {
final Logger LOG = Logger.getLogger(V1Controller.class.getName());
- final Set<String> completeConfigParam = getConfigKeys(config,
KUBERNETES_VOLUME_CLAIM_PREFIX);
- final int prefixLength = KUBERNETES_VOLUME_CLAIM_PREFIX.length();
+ final String prefixKey = String.format(KUBERNETES_VOLUME_CLAIM_PREFIX,
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME :
KubernetesConstants.MANAGER_NAME);
+ final Set<String> completeConfigParam = getConfigKeys(config, prefixKey);
+ final int prefixLength = prefixKey.length();
final int volumeNameIdx = 0;
final int optionIdx = 1;
final Matcher matcher =
KubernetesConstants.VALID_LOWERCASE_RFC_1123_REGEX.matcher("");
diff --git
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
index e6fc819..96793b6 100644
---
a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
+++
b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
@@ -99,13 +99,10 @@ public class V1Controller extends KubernetesController {
private final AppsV1Api appsClient;
private final CoreV1Api coreClient;
- private Map<String, Map<KubernetesConstants.VolumeClaimTemplateConfigKeys,
String>>
- persistentVolumeClaimConfigs = null;
-
V1Controller(Config configuration, Config runtimeConfiguration) {
super(configuration, runtimeConfiguration);
- isPodTemplateDisabled =
KubernetesContext.getPodTemplateConfigMapDisabled(configuration);
+ isPodTemplateDisabled =
KubernetesContext.getPodTemplateDisabled(configuration);
LOG.log(Level.WARNING, String.format("Custom Pod Templates are %s",
isPodTemplateDisabled ? "DISABLED" : "ENABLED"));
@@ -124,7 +121,7 @@ public class V1Controller extends KubernetesController {
boolean submit(PackingPlan packingPlan) {
final String topologyName = getTopologyName();
if (!topologyName.equals(topologyName.toLowerCase())) {
- throw new TopologySubmissionException("K8S scheduler does not allow
upper case topologies.");
+ throw new TopologySubmissionException("K8S scheduler does not allow
upper case topology's.");
}
final Resource containerResource = getContainerResource(packingPlan);
@@ -138,28 +135,19 @@ public class V1Controller extends KubernetesController {
throw new TopologySubmissionException(e.getMessage());
}
- // Get and then create Persistent Volume Claims from the CLI.
- persistentVolumeClaimConfigs =
- KubernetesContext.getVolumeClaimTemplates(getConfiguration());
- if (KubernetesContext.getPersistentVolumeClaimDisabled(getConfiguration())
- && !persistentVolumeClaimConfigs.isEmpty()) {
- final String message =
- String.format("Configuring Persistent Volume Claim from CLI is
disabled: '%s'",
- topologyName);
- LOG.log(Level.WARNING, message);
- throw new TopologySubmissionException(message);
- }
-
- // find the max number of instances in a container so we can open
+ // Find the max number of instances in a container so that we can open
// enough ports if remote debugging is enabled.
int numberOfInstances = 0;
for (PackingPlan.ContainerPlan containerPlan :
packingPlan.getContainers()) {
numberOfInstances = Math.max(numberOfInstances,
containerPlan.getInstances().size());
}
- final V1StatefulSet statefulSet = createStatefulSet(containerResource,
numberOfInstances);
+ final V1StatefulSet executors = createStatefulSet(containerResource,
numberOfInstances, true);
+ final V1StatefulSet manager = createStatefulSet(containerResource,
numberOfInstances, false);
try {
- appsClient.createNamespacedStatefulSet(getNamespace(), statefulSet, null,
+ appsClient.createNamespacedStatefulSet(getNamespace(), executors, null,
+ null, null);
+ appsClient.createNamespacedStatefulSet(getNamespace(), manager, null,
null, null);
} catch (ApiException e) {
final String message = String.format("Error creating topology: %s%n",
e.getResponseBody());
@@ -173,7 +161,7 @@ public class V1Controller extends KubernetesController {
@Override
boolean killTopology() {
removePersistentVolumeClaims();
- deleteStatefulSet();
+ deleteStatefulSets();
deleteService();
return true;
}
@@ -186,9 +174,8 @@ public class V1Controller extends KubernetesController {
@Override
boolean restart(int shardId) {
try {
- coreClient.deleteCollectionNamespacedPod(getNamespace(), null, null,
null, null,
- 0, createTopologySelectorLabels(), null, null, null, null,
- null, null, null);
+ coreClient.deleteCollectionNamespacedPod(getNamespace(), null, null,
null, null, 0,
+ createTopologySelectorLabels(), null, null, null, null, null, null,
null);
LOG.log(Level.WARNING, String.format("Restarting topology '%s'...",
getTopologyName()));
} catch (ApiException e) {
LOG.log(Level.SEVERE, String.format("Failed to restart topology
'%s'...", getTopologyName()));
@@ -245,14 +232,14 @@ public class V1Controller extends KubernetesController {
private void patchStatefulSetReplicas(int replicas) throws ApiException {
final String body =
- String.format(JSON_PATCH_STATEFUL_SET_REPLICAS_FORMAT,
+
String.format(KubernetesConstants.JSON_PATCH_STATEFUL_SET_REPLICAS_FORMAT,
replicas);
final V1Patch patch = new V1Patch(body);
PatchUtils.patch(V1StatefulSet.class,
() ->
appsClient.patchNamespacedStatefulSetCall(
- getTopologyName(),
+ getStatefulSetName(true),
getNamespace(),
patch,
null,
@@ -264,11 +251,8 @@ public class V1Controller extends KubernetesController {
appsClient.getApiClient());
}
- private static final String JSON_PATCH_STATEFUL_SET_REPLICAS_FORMAT =
- "[{\"op\":\"replace\",\"path\":\"/spec/replicas\",\"value\":%d}]";
-
V1StatefulSet getStatefulSet() throws ApiException {
- return appsClient.readNamespacedStatefulSet(getTopologyName(),
getNamespace(),
+ return appsClient.readNamespacedStatefulSet(getStatefulSetName(true),
getNamespace(),
null, null, null);
}
@@ -307,18 +291,19 @@ public class V1Controller extends KubernetesController {
+ "] in namespace [" + getNamespace() + "] is deleted.");
}
- void deleteStatefulSet() {
- try (Response response =
appsClient.deleteNamespacedStatefulSetCall(getTopologyName(),
- getNamespace(), null, null, 0, null,
- KubernetesConstants.DELETE_OPTIONS_PROPAGATION_POLICY, null,
null).execute()) {
+ void deleteStatefulSets() {
+ try (Response response =
appsClient.deleteCollectionNamespacedStatefulSetCall(getNamespace(),
+ null, null, null, null, null, createTopologySelectorLabels(), null,
null, null, null, null,
+ null, null, null)
+ .execute()) {
if (!response.isSuccessful()) {
if (response.code() == HTTP_NOT_FOUND) {
- LOG.log(Level.WARNING, "Tried to delete a non-existent StatefulSet
for Topology: "
+ LOG.log(Level.WARNING, "Tried to delete a non-existent StatefulSets
for Topology: "
+ getTopologyName());
return;
}
- LOG.log(Level.SEVERE, "Error when deleting the StatefulSet of the job
["
+ LOG.log(Level.SEVERE, "Error when deleting the StatefulSets of the job
["
+ getTopologyName() + "] in namespace [" + getNamespace() +
"]");
LOG.log(Level.SEVERE, "Error killing topology message: " +
response.message());
KubernetesUtils.logResponseBodyIfPresent(LOG, response);
@@ -333,16 +318,24 @@ public class V1Controller extends KubernetesController {
return;
}
throw new TopologyRuntimeManagementException("Error deleting topology ["
- + getTopologyName() + "] Kubernetes StatefulSet", e);
+ + getTopologyName() + "] Kubernetes StatefulSets", e);
} catch (IOException e) {
throw new TopologyRuntimeManagementException("Error deleting topology ["
- + getTopologyName() + "] Kubernetes StatefulSet", e);
+ + getTopologyName() + "] Kubernetes StatefulSets", e);
}
LOG.log(Level.INFO, "StatefulSet for the Job [" + getTopologyName()
+ "] in namespace [" + getNamespace() + "] is deleted.");
}
- protected List<String> getExecutorCommand(String containerId, int
numOfInstances) {
+ /**
+ * Generates the command to start Heron within the <code>container</code>.
+ * @param containerId Passed down to <>SchedulerUtils</> to generate
executor command.
+ * @param numOfInstances Used to configure the debugging ports.
+ * @param isExecutor Flag used to generate the correct <code>shard_id</code>.
+ * @return The complete command to start Heron in a <code>container</code>.
+ */
+ protected List<String> getExecutorCommand(String containerId, int
numOfInstances,
+ boolean isExecutor) {
final Config configuration = getConfiguration();
final Config runtimeConfiguration = getRuntimeConfiguration();
final Map<ExecutorPort, String> ports =
@@ -370,13 +363,22 @@ public class V1Controller extends KubernetesController {
"-c",
KubernetesUtils.getConfCommand(configuration)
+ " && " + KubernetesUtils.getFetchCommand(configuration,
runtimeConfiguration)
- + " && " + setShardIdEnvironmentVariableCommand()
+ + " && " + setShardIdEnvironmentVariableCommand(isExecutor)
+ " && " + String.join(" ", executorCommand)
);
}
- private static String setShardIdEnvironmentVariableCommand() {
- return String.format("%s=${POD_NAME##*-} && echo shardId=${%s}",
ENV_SHARD_ID, ENV_SHARD_ID);
+ /**
+ * Configures the <code>shard_id</code> for the Heron container based on
whether it is an <code>executor</code>
+ * or <code>manager</code>. <code>executor</code> IDs are [1 - n) and the
<code>manager</code> IDs start at 0.
+ * @param isExecutor Switch flag to generate correct command.
+ * @return The command required to put the Heron instance in
<code>executor</code> or <code>manager</code> mode.
+ */
+ @VisibleForTesting
+ protected static String setShardIdEnvironmentVariableCommand(boolean
isExecutor) {
+ final String pattern = String.format("%%s=%s && echo shardId=${%%s}",
+ isExecutor ? "$((${POD_NAME##*-} + 1))" : "${POD_NAME##*-}");
+ return String.format(pattern, ENV_SHARD_ID, ENV_SHARD_ID);
}
private V1Service createTopologyService() {
@@ -401,21 +403,45 @@ public class V1Controller extends KubernetesController {
return service;
}
- private V1StatefulSet createStatefulSet(Resource containerResource, int
numberOfInstances) {
+ /**
+ * Creates and configures the <code>StatefulSet</code> which the topology's
<code>executor</code>s will run in.
+ * @param containerResource Passed down to configure the
<code>executor</code> resource limits.
+ * @param numberOfInstances Used to configure the execution command and
ports for the <code>executor</code>.
+ * @param isExecutor Flag used to configure components specific to
<code>executor</code> and <code>manager</code>.
+ * @return A fully configured <code>StatefulSet</code> for the topology's
<code>executors</code>.
+ */
+ private V1StatefulSet createStatefulSet(Resource containerResource, int
numberOfInstances,
+ boolean isExecutor) {
final String topologyName = getTopologyName();
final Config runtimeConfiguration = getRuntimeConfiguration();
+ // Get and then create Persistent Volume Claims from the CLI.
+ final Map<String, Map<KubernetesConstants.VolumeClaimTemplateConfigKeys,
String>> configsPVC =
+ KubernetesContext.getVolumeClaimTemplates(getConfiguration(),
isExecutor);
+ if (KubernetesContext.getPersistentVolumeClaimDisabled(getConfiguration())
+ && !configsPVC.isEmpty()) {
+ final String message =
+ String.format("'%s': Configuring Persistent Volume Claim from CLI is
disabled",
+ topologyName);
+ LOG.log(Level.WARNING, message);
+ throw new TopologySubmissionException(message);
+ }
+
final V1StatefulSet statefulSet = new V1StatefulSet();
// Setup StatefulSet's metadata.
final V1ObjectMeta objectMeta = new V1ObjectMeta()
- .name(topologyName);
+ .name(getStatefulSetName(isExecutor))
+ .labels(getPodLabels(topologyName));
statefulSet.setMetadata(objectMeta);
- // Create the stateful set spec.
+ // Create the StatefulSet Spec.
+ // Reduce replica count by one for Executors and set to one for Manager.
+ final int replicasCount =
+ isExecutor ? Runtime.numContainers(runtimeConfiguration).intValue() -
1 : 1;
final V1StatefulSetSpec statefulSetSpec = new V1StatefulSetSpec()
.serviceName(topologyName)
- .replicas(Runtime.numContainers(runtimeConfiguration).intValue());
+ .replicas(replicasCount);
// Parallel pod management tells the StatefulSet controller to launch or
terminate
// all Pods in parallel, and not to wait for Pods to become Running and
Ready or completely
@@ -428,10 +454,10 @@ public class V1Controller extends KubernetesController {
.matchLabels(getPodMatchLabels(topologyName));
statefulSetSpec.setSelector(selector);
- // create a pod template
- final V1PodTemplateSpec podTemplateSpec = loadPodFromTemplate();
+ // Create a Pod Template.
+ final V1PodTemplateSpec podTemplateSpec = loadPodFromTemplate(isExecutor);
- // set up pod meta
+ // Set up Pod Metadata.
final V1ObjectMeta templateMetaData = new
V1ObjectMeta().labels(getPodLabels(topologyName));
Map<String, String> annotations = new HashMap<>();
annotations.putAll(getPodAnnotations());
@@ -439,27 +465,23 @@ public class V1Controller extends KubernetesController {
templateMetaData.setAnnotations(annotations);
podTemplateSpec.setMetadata(templateMetaData);
- final List<String> command = getExecutorCommand("$" + ENV_SHARD_ID,
numberOfInstances);
- configurePodSpec(podTemplateSpec, command, containerResource,
numberOfInstances);
+ configurePodSpec(podTemplateSpec, containerResource, numberOfInstances,
isExecutor, configsPVC);
statefulSetSpec.setTemplate(podTemplateSpec);
statefulSet.setSpec(statefulSetSpec);
- statefulSetSpec.setVolumeClaimTemplates(
- createPersistentVolumeClaims(persistentVolumeClaimConfigs));
+
statefulSetSpec.setVolumeClaimTemplates(createPersistentVolumeClaims(configsPVC));
return statefulSet;
}
private Map<String, String> getPodAnnotations() {
- Config config = getConfiguration();
- return KubernetesContext.getPodAnnotations(config);
+ return KubernetesContext.getPodAnnotations(getConfiguration());
}
private Map<String, String> getServiceAnnotations() {
- Config config = getConfiguration();
- return KubernetesContext.getServiceAnnotations(config);
+ return KubernetesContext.getServiceAnnotations(getConfiguration());
}
private Map<String, String> getPrometheusAnnotations() {
@@ -490,51 +512,60 @@ public class V1Controller extends KubernetesController {
return KubernetesContext.getServiceLabels(getConfiguration());
}
- private void configurePodSpec(final V1PodTemplateSpec podTemplateSpec,
- List<String> executorCommand,
- Resource resource,
- int numberOfInstances) {
+ /**
+ * Configures the <code>Pod Spec</code> section of the
<code>StatefulSet</code>. The <code>Heron</code> container
+ * will be configured to allow it to function but other supplied containers
are loaded verbatim.
+ * @param podTemplateSpec The <code>Pod Template Spec</code> section to
update.
+ * @param resource Passed down to configure the resource limits.
+ * @param numberOfInstances Passed down to configure the ports.
+ * @param isExecutor Flag used to configure components specific to
<code>Executor</code> and <code>Manager</code>.
+ * @param configPVC <code>Persistent Volume Claim</code> configurations
options.
+ */
+ private void configurePodSpec(final V1PodTemplateSpec podTemplateSpec,
Resource resource,
+ int numberOfInstances, boolean isExecutor,
+ final Map<String, Map<KubernetesConstants.VolumeClaimTemplateConfigKeys,
String>> configPVC) {
if (podTemplateSpec.getSpec() == null) {
podTemplateSpec.setSpec(new V1PodSpec());
}
final V1PodSpec podSpec = podTemplateSpec.getSpec();
- // set the termination period to 0 so pods can be deleted quickly
+ // Set the termination period to 0 so pods can be deleted quickly
podSpec.setTerminationGracePeriodSeconds(0L);
- // set the pod tolerations so pods are rescheduled when nodes go down
+ // Set the pod tolerations so pods are rescheduled when nodes go down
//
https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions
configureTolerations(podSpec);
- // Get <executor> container and discard all others.
- final String executorName = KubernetesConstants.EXECUTOR_NAME;
- V1Container executorContainer = null;
+ // Get <Heron> container and ignore all others.
+ final String containerName =
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME :
KubernetesConstants.MANAGER_NAME;
+ V1Container heronContainer = null;
List<V1Container> containers = podSpec.getContainers();
if (containers != null) {
for (V1Container container : containers) {
final String name = container.getName();
- if (name != null && name.equals(executorName)) {
- if (executorContainer != null) {
+ if (name != null && name.equals(containerName)) {
+ if (heronContainer != null) {
throw new TopologySubmissionException(
- String.format("Multiple configurations found for %s
container", executorName));
+ String.format("Multiple configurations found for '%s'
container", containerName));
}
- executorContainer = container;
+ heronContainer = container;
}
}
} else {
containers = new LinkedList<>();
}
- if (executorContainer == null) {
- executorContainer = new V1Container().name(executorName);
- containers.add(executorContainer);
+ if (heronContainer == null) {
+ heronContainer = new V1Container().name(containerName);
+ containers.add(heronContainer);
}
- if (!persistentVolumeClaimConfigs.isEmpty()) {
- configurePodWithPersistentVolumeClaimVolumesAndMounts(podSpec,
executorContainer);
+ if (!configPVC.isEmpty()) {
+ configurePodWithPersistentVolumeClaimVolumesAndMounts(podSpec,
heronContainer, configPVC);
}
- configureExecutorContainer(executorCommand, resource, numberOfInstances,
executorContainer);
+ configureHeronContainer(resource, numberOfInstances, heronContainer,
isExecutor);
podSpec.setContainers(containers);
@@ -543,6 +574,10 @@ public class V1Controller extends KubernetesController {
mountSecretsAsVolumes(podSpec);
}
+ /**
+ * Adds <code>tolerations</code> to the <code>Pod Spec</code> with Heron's
values taking precedence.
+ * @param spec <code>Pod Spec</code> to be configured.
+ */
@VisibleForTesting
protected void configureTolerations(final V1PodSpec spec) {
KubernetesUtils.V1ControllerUtils<V1Toleration> utils =
@@ -553,6 +588,10 @@ public class V1Controller extends KubernetesController {
);
}
+ /**
+ * Generates a list of <code>tolerations</code> which Heron requires.
+ * @return A list of configured <code>tolerations</code>.
+ */
@VisibleForTesting
protected static List<V1Toleration> getTolerations() {
final List<V1Toleration> tolerations = new ArrayList<>();
@@ -569,6 +608,10 @@ public class V1Controller extends KubernetesController {
return tolerations;
}
+ /**
+ * Adds volume to the <code>Pod Spec</code> that Heron requires. Heron's
values taking precedence.
+ * @param spec <code>Pod Spec</code> to be configured.
+ */
@VisibleForTesting
protected void addVolumesIfPresent(final V1PodSpec spec) {
final Config config = getConfiguration();
@@ -606,15 +649,24 @@ public class V1Controller extends KubernetesController {
}
}
- private void configureExecutorContainer(List<String> executorCommand,
Resource resource,
- int numberOfInstances, final
V1Container container) {
+ /**
+ * Configures the <code>Heron</code> container with values for parameters
Heron requires for functioning.
+ * @param resource Resource limits.
+ * @param numberOfInstances Required number of <code>executor</code>
containers which is used to configure ports.
+ * @param container The <code>executor</code> container to be configured.
+ * @param isExecutor Flag indicating whether to set a <code>executor</code>
or <code>manager</code> command.
+ */
+ private void configureHeronContainer(Resource resource, int
numberOfInstances,
+ final V1Container container, boolean
isExecutor) {
final Config configuration = getConfiguration();
// Set up the container images.
container.setImage(KubernetesContext.getExecutorDockerImage(configuration));
// Set up the container command.
- container.setCommand(executorCommand);
+ final List<String> command =
+ getExecutorCommand("$" + ENV_SHARD_ID, numberOfInstances, isExecutor);
+ container.setCommand(command);
if (KubernetesContext.hasImagePullPolicy(configuration)) {
container.setImagePullPolicy(KubernetesContext.getKubernetesImagePullPolicy(configuration));
@@ -627,7 +679,7 @@ public class V1Controller extends KubernetesController {
setSecretKeyRefs(container);
// Set container resources
- configureContainerResources(container, configuration, resource);
+ configureContainerResources(container, configuration, resource,
isExecutor);
// Set container ports.
final boolean debuggingEnabled =
@@ -639,30 +691,60 @@ public class V1Controller extends KubernetesController {
mountVolumeIfPresent(container);
}
+ /**
+ * Configures the resources in the <code>container</code> with values in the
<code>config</code> taking precedence.
+ * @param container The <code>container</code> to be configured.
+ * @param configuration The <code>Config</code> object to check if a
resource request needs to be set.
+ * @param resource User defined resources limits from input.
+ * @param isExecutor
+ */
@VisibleForTesting
protected void configureContainerResources(final V1Container container,
- final Config configuration, final
Resource resource) {
+ final Config configuration, final
Resource resource,
+ boolean isExecutor) {
if (container.getResources() == null) {
container.setResources(new V1ResourceRequirements());
}
final V1ResourceRequirements resourceRequirements =
container.getResources();
- // Configure resource limits. Deduplicate on limit name with user values
taking precedence.
+ // Collect Limits and Requests from CLI.
+ final Map<String, Quantity> limitsCLI = createResourcesRequirement(
+ KubernetesContext.getResourceLimits(configuration, isExecutor));
+ final Map<String, Quantity> requestsCLI = createResourcesRequirement(
+ KubernetesContext.getResourceRequests(configuration, isExecutor));
+
if (resourceRequirements.getLimits() == null) {
resourceRequirements.setLimits(new HashMap<>());
}
+
+ // Set Limits and Resources from CLI <if> available, <else> use Configs.
Deduplicate on name
+ // with precedence [1] CLI, [2] Config.
final Map<String, Quantity> limits = resourceRequirements.getLimits();
- limits.put(KubernetesConstants.MEMORY,
- Quantity.fromString(KubernetesUtils.Megabytes(
- resource.getRam())));
- limits.put(KubernetesConstants.CPU,
- Quantity.fromString(Double.toString(roundDecimal(
- resource.getCpu(), 3))));
+ final Quantity limitCPU = limitsCLI.getOrDefault(KubernetesConstants.CPU,
+ Quantity.fromString(Double.toString(roundDecimal(resource.getCpu(),
3))));
+ final Quantity limitMEMORY =
limitsCLI.getOrDefault(KubernetesConstants.MEMORY,
+ Quantity.fromString(KubernetesUtils.Megabytes(resource.getRam())));
+
+ limits.put(KubernetesConstants.MEMORY, limitMEMORY);
+ limits.put(KubernetesConstants.CPU, limitCPU);
// Set the Kubernetes container resource request.
+ // Order: [1] CLI, [2] EQUAL_TO_LIMIT, [3] NOT_SET
KubernetesContext.KubernetesResourceRequestMode requestMode =
KubernetesContext.getKubernetesRequestMode(configuration);
- if (requestMode ==
KubernetesContext.KubernetesResourceRequestMode.EQUAL_TO_LIMIT) {
+ if (!requestsCLI.isEmpty()) {
+ if (resourceRequirements.getRequests() == null) {
+ resourceRequirements.setRequests(new HashMap<>());
+ }
+ final Map<String, Quantity> requests =
resourceRequirements.getRequests();
+
+ if (requestsCLI.containsKey(KubernetesConstants.MEMORY)) {
+ requests.put(KubernetesConstants.MEMORY,
requestsCLI.get(KubernetesConstants.MEMORY));
+ }
+ if (requestsCLI.containsKey(KubernetesConstants.CPU)) {
+ requests.put(KubernetesConstants.CPU,
requestsCLI.get(KubernetesConstants.CPU));
+ }
+ } else if (requestMode ==
KubernetesContext.KubernetesResourceRequestMode.EQUAL_TO_LIMIT) {
LOG.log(Level.CONFIG, "Setting K8s Request equal to Limit");
resourceRequirements.setRequests(limits);
} else {
@@ -671,6 +753,38 @@ public class V1Controller extends KubernetesController {
container.setResources(resourceRequirements);
}
+ /**
+ * Creates <code>Resource Requirements</code> from a Map of
<code>Config</code> items for <code>CPU</code>
+ * and <code>Memory</code>.
+ * @param configs <code>Configs</code> to be parsed for configuration.
+ * @return Configured <code>Resource Requirements</code>. An
<code>empty</code> map will be returned
+ * if there are no <code>configs</code>.
+ */
+ @VisibleForTesting
+ protected Map<String, Quantity> createResourcesRequirement(Map<String,
String> configs) {
+ final Map<String, Quantity> requirements = new HashMap<>();
+
+ if (configs == null || configs.isEmpty()) {
+ return requirements;
+ }
+
+ final String memoryLimit = configs.get(KubernetesConstants.MEMORY);
+ if (memoryLimit != null && !memoryLimit.isEmpty()) {
+ requirements.put(KubernetesConstants.MEMORY,
Quantity.fromString(memoryLimit));
+ }
+ final String cpuLimit = configs.get(KubernetesConstants.CPU);
+ if (cpuLimit != null && !cpuLimit.isEmpty()) {
+ requirements.put(KubernetesConstants.CPU, Quantity.fromString(cpuLimit));
+ }
+
+ return requirements;
+ }
+
+ /**
+ * Configures the environment variables in the <code>container</code> with
those Heron requires.
+ * Heron's values take precedence.
+ * @param container The <code>container</code> to be configured.
+ */
@VisibleForTesting
protected void configureContainerEnvVars(final V1Container container) {
// Deduplicate on var name with Heron defaults take precedence.
@@ -681,6 +795,10 @@ public class V1Controller extends KubernetesController {
);
}
+ /**
+ * Generates a list of <code>Environment Variables</code> required by Heron
to function.
+ * @return A list of configured <code>Environment Variables</code> required
by Heron to function.
+ */
@VisibleForTesting
protected static List<V1EnvVar> getExecutorEnvVars() {
final V1EnvVar envVarHost = new V1EnvVar();
@@ -698,6 +816,12 @@ public class V1Controller extends KubernetesController {
return Arrays.asList(envVarHost, envVarPodName);
}
+ /**
+ * Configures the ports in the <code>container</code> with those Heron
requires. Heron's values take precedence.
+ * @param remoteDebugEnabled Flag used to indicate if debugging ports need
to be added.
+ * @param numberOfInstances The number of debugging ports to be opened.
+ * @param container <code>container</code> to be configured.
+ */
@VisibleForTesting
protected void configureContainerPorts(boolean remoteDebugEnabled, int
numberOfInstances,
final V1Container container) {
@@ -716,6 +840,10 @@ public class V1Controller extends KubernetesController {
);
}
+ /**
+ * Generates a list of <code>ports</code> required by Heron to function.
+ * @return A list of configured <code>ports</code> required by Heron to
function.
+ */
@VisibleForTesting
protected static List<V1ContainerPort> getExecutorPorts() {
List<V1ContainerPort> ports = new LinkedList<>();
@@ -728,6 +856,11 @@ public class V1Controller extends KubernetesController {
return ports;
}
+ /**
+ * Generate the debugging ports required by Heron.
+ * @param numberOfInstances The number of debugging ports to generate.
+ * @return A list of configured debugging <code>ports</code>.
+ */
@VisibleForTesting
protected static List<V1ContainerPort> getDebuggingPorts(int
numberOfInstances) {
List<V1ContainerPort> ports = new LinkedList<>();
@@ -742,6 +875,10 @@ public class V1Controller extends KubernetesController {
return ports;
}
+ /**
+ * Adds volume mounts to the <code>container</code> that Heron requires.
Heron's values taking precedence.
+ * @param container <code>container</code> to be configured.
+ */
@VisibleForTesting
protected void mountVolumeIfPresent(final V1Container container) {
final Config config = getConfiguration();
@@ -789,9 +926,16 @@ public class V1Controller extends KubernetesController {
return Math.round(value * scale) / scale;
}
+ /**
+ * Initiates the process of locating and loading <code>Pod Template</code>
from a <code>ConfigMap</code>.
+ * The loaded text is then parsed into a usable <code>Pod Template</code>.
+ * @param isExecutor Flag to indicate loading of <code>Pod Template</code>
for <code>Executor</code>
+ * or <code>Manager</code>.
+ * @return A <code>Pod Template</code> which is loaded and parsed from a
<code>ConfigMap</code>.
+ */
@VisibleForTesting
- protected V1PodTemplateSpec loadPodFromTemplate() {
- final Pair<String, String> podTemplateConfigMapName =
getPodTemplateLocation();
+ protected V1PodTemplateSpec loadPodFromTemplate(boolean isExecutor) {
+ final Pair<String, String> podTemplateConfigMapName =
getPodTemplateLocation(isExecutor);
// Default Pod Template.
if (podTemplateConfigMapName == null) {
@@ -838,10 +982,16 @@ public class V1Controller extends KubernetesController {
}
}
+ /**
+ * Extracts the <code>ConfigMap</code> and <code>Pod Template</code> names
from the CLI parameter.
+ * @param isExecutor Flag to indicate loading of <code>Pod Template</code>
for <code>Executor</code>
+ * or <code>Manager</code>.
+ * @return A pair of the form <code>(ConfigMap, Pod Template)</code>.
+ */
@VisibleForTesting
- protected Pair<String, String> getPodTemplateLocation() {
+ protected Pair<String, String> getPodTemplateLocation(boolean isExecutor) {
final String podTemplateConfigMapName = KubernetesContext
- .getPodTemplateConfigMapName(getConfiguration());
+ .getPodTemplateConfigMapName(getConfiguration(), isExecutor);
if (podTemplateConfigMapName == null) {
return null;
@@ -864,6 +1014,11 @@ public class V1Controller extends KubernetesController {
}
}
+ /**
+ * Retrieves a <code>ConfigMap</code> from the K8s cluster in the API
Server's namespace.
+ * @param configMapName Name of the <code>ConfigMap</code> to retrieve.
+ * @return The retrieved <code>ConfigMap</code>.
+ */
@VisibleForTesting
protected V1ConfigMap getConfigMap(String configMapName) {
try {
@@ -996,12 +1151,14 @@ public class V1Controller extends KubernetesController {
* Makes a call to generate <code>Volumes</code> and <code>Volume
Mounts</code> and then inserts them.
* @param podSpec All generated <code>V1Volume</code> will be placed in the
<code>Pod Spec</code>.
* @param executor All generated <code>V1VolumeMount</code> will be placed
in the <code>Container</code>.
+ * @param configPVC <code>Persistent Volume Claim</code> configurations
options.
*/
@VisibleForTesting
protected void configurePodWithPersistentVolumeClaimVolumesAndMounts(final
V1PodSpec podSpec,
- final
V1Container executor) {
+ final V1Container executor,
+ final Map<String, Map<KubernetesConstants.VolumeClaimTemplateConfigKeys,
String>> configPVC) {
Pair<List<V1Volume>, List<V1VolumeMount>> volumesAndMounts =
-
createPersistentVolumeClaimVolumesAndMounts(persistentVolumeClaimConfigs);
+ createPersistentVolumeClaimVolumesAndMounts(configPVC);
// Deduplicate on Names with Persistent Volume Claims taking precedence.
@@ -1072,7 +1229,7 @@ public class V1Controller extends KubernetesController {
}
/**
- * Generates the <code>Label</code> which are attached to a Topologies
Persistent Volume Claims.
+ * Generates the <code>Label</code> which are attached to a Topology's
Persistent Volume Claims.
* @param topologyName Attached to the topology match label.
* @return A map consisting of the <code>label-value</code> pairs to be used
in <code>Label</code>s.
*/
@@ -1087,6 +1244,17 @@ public class V1Controller extends KubernetesController {
}
/**
+ * Generates the <code>StatefulSet</code> name depending on if it is a
<code>Executor</code> or
+ * <code>Manager</code>.
+ * @param isExecutor Flag used to generate name for <code>Executor</code> or
<code>Manager</code>.
+ * @return String <code>"topology-name"-executors</code>.
+ */
+ private String getStatefulSetName(boolean isExecutor) {
+ return String.format("%s-%s", getTopologyName(),
+ isExecutor ? KubernetesConstants.EXECUTOR_NAME + "s" :
KubernetesConstants.MANAGER_NAME);
+ }
+
+ /**
* Generates the <code>Selector</code> match labels with which resources in
this topology can be found.
* @return A label of the form <code>app=heron,topology=topology-name</code>.
*/
diff --git
a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesContextTest.java
b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesContextTest.java
index 95de4a8..988456f 100644
---
a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesContextTest.java
+++
b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesContextTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
+import org.apache.heron.common.basics.Pair;
import org.apache.heron.scheduler.TopologySubmissionException;
import org.apache.heron.scheduler.kubernetes.KubernetesUtils.TestTuple;
import org.apache.heron.spi.common.Config;
@@ -36,38 +37,48 @@ import static
org.apache.heron.scheduler.kubernetes.KubernetesConstants.VolumeCl
public class KubernetesContextTest {
private static final String TOPOLOGY_NAME = "Topology-Name";
- private static final String KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME =
- "heron.kubernetes.pod.template.configmap.name";
private static final String POD_TEMPLATE_CONFIGMAP_NAME =
"pod-template-configmap-name";
private final Config config = Config.newBuilder().build();
private final Config configWithPodTemplateConfigMap = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
+ .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
POD_TEMPLATE_CONFIGMAP_NAME)
.build();
@Test
public void testPodTemplateConfigMapName() {
-
Assert.assertEquals(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
- KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME);
- Assert.assertEquals(
-
KubernetesContext.getPodTemplateConfigMapName(configWithPodTemplateConfigMap),
- POD_TEMPLATE_CONFIGMAP_NAME);
- Assert.assertNull(KubernetesContext.getPodTemplateConfigMapName(config));
+ final String executorKey =
String.format(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
+ KubernetesConstants.EXECUTOR_NAME);
+ final Config configExecutor = Config.newBuilder()
+ .put(executorKey, POD_TEMPLATE_CONFIGMAP_NAME)
+ .build();
+ Assert.assertEquals("Executor location", POD_TEMPLATE_CONFIGMAP_NAME,
+ KubernetesContext.getPodTemplateConfigMapName(configExecutor, true));
+
+ final String managerKey =
String.format(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
+ KubernetesConstants.MANAGER_NAME);
+ final Config configManager = Config.newBuilder()
+ .put(managerKey, POD_TEMPLATE_CONFIGMAP_NAME)
+ .build();
+ Assert.assertEquals("Manager location", POD_TEMPLATE_CONFIGMAP_NAME,
+ KubernetesContext.getPodTemplateConfigMapName(configManager, false));
+
+ Assert.assertNull("No Pod Template",
+ KubernetesContext.getPodTemplateConfigMapName(config, true));
}
@Test
- public void testPodTemplateConfigMapDisabled() {
-
Assert.assertFalse(KubernetesContext.getPodTemplateConfigMapDisabled(config));
+ public void testPodTemplateDisabled() {
+ Assert.assertFalse(KubernetesContext.getPodTemplateDisabled(config));
Assert.assertFalse(KubernetesContext
- .getPodTemplateConfigMapDisabled(configWithPodTemplateConfigMap));
+ .getPodTemplateDisabled(configWithPodTemplateConfigMap));
final Config configWithPodTemplateConfigMapOff = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
+ .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
POD_TEMPLATE_CONFIGMAP_NAME)
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_DISABLED,
"TRUE")
+ .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_DISABLED, "TRUE")
.build();
Assert.assertTrue(KubernetesContext
- .getPodTemplateConfigMapDisabled(configWithPodTemplateConfigMapOff));
+ .getPodTemplateDisabled(configWithPodTemplateConfigMapOff));
}
@Test
@@ -77,7 +88,7 @@ public class KubernetesContextTest {
.getPersistentVolumeClaimDisabled(configWithPodTemplateConfigMap));
final Config configWithPodTemplateConfigMapOff = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
+ .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
POD_TEMPLATE_CONFIGMAP_NAME)
.put(KubernetesContext.KUBERNETES_PERSISTENT_VOLUME_CLAIMS_CLI_DISABLED, "TRUE")
.build();
@@ -90,54 +101,88 @@ public class KubernetesContextTest {
final String volumeNameOne = "volume-name-one";
final String volumeNameTwo = "volume-name-two";
final String claimName = "OnDeMaNd";
- final String keyPattern = KubernetesContext.KUBERNETES_VOLUME_CLAIM_PREFIX
+ "%s.%s";
+ final String keyPattern = KubernetesContext.KUBERNETES_VOLUME_CLAIM_PREFIX
+ "%%s.%%s";
+ final String keyExecutor = String.format(keyPattern,
KubernetesConstants.EXECUTOR_NAME);
+ final String keyManager = String.format(keyPattern,
KubernetesConstants.MANAGER_NAME);
final String storageClassField =
VolumeClaimTemplateConfigKeys.storageClassName.name();
final String pathField = VolumeClaimTemplateConfigKeys.path.name();
final String claimNameField =
VolumeClaimTemplateConfigKeys.claimName.name();
final String expectedStorageClass = "expected-storage-class";
- final String storageClassKeyOne = String.format(keyPattern, volumeNameOne,
storageClassField);
- final String storageClassKeyTwo = String.format(keyPattern, volumeNameTwo,
storageClassField);
final String expectedPath = "/path/for/volume/expected";
- final String pathKeyOne = String.format(keyPattern, volumeNameOne,
pathField);
- final String pathKeyTwo = String.format(keyPattern, volumeNameTwo,
pathField);
- final String claimNameKeyOne = String.format(keyPattern, volumeNameOne,
claimNameField);
- final String claimNameKeyTwo = String.format(keyPattern, volumeNameTwo,
claimNameField);
-
- final Config configPVC = Config.newBuilder()
- .put(pathKeyOne, expectedPath)
- .put(pathKeyTwo, expectedPath)
- .put(claimNameKeyOne, claimName)
- .put(claimNameKeyTwo, claimName)
- .put(storageClassKeyOne, expectedStorageClass)
- .put(storageClassKeyTwo, expectedStorageClass)
- .build();
- final List<String> expectedKeys = Arrays.asList(volumeNameOne,
volumeNameTwo);
- final List<VolumeClaimTemplateConfigKeys> expectedOptionsKeys =
- Arrays.asList(VolumeClaimTemplateConfigKeys.path,
- VolumeClaimTemplateConfigKeys.storageClassName,
- VolumeClaimTemplateConfigKeys.claimName);
- final List<String> expectedOptionsValues =
- Arrays.asList(expectedPath, expectedStorageClass, claimName);
-
- // List of provided PVC options.
- final Map<String, Map<VolumeClaimTemplateConfigKeys, String>> mapOfPVC =
- KubernetesContext.getVolumeClaimTemplates(configPVC);
-
- Assert.assertTrue("Contains all provided Volumes",
- mapOfPVC.keySet().containsAll(expectedKeys));
- for (Map<VolumeClaimTemplateConfigKeys, String> items : mapOfPVC.values())
{
- Assert.assertTrue("Contains all provided option keys",
- items.keySet().containsAll(expectedOptionsKeys));
- Assert.assertTrue("Contains all provided option values",
- items.values().containsAll(expectedOptionsValues));
+
+ // Test case container.
+ // Input: Config to extract options from, Boolean to indicate
Manager/Executor.
+ // Output: [0] expectedKeys, [1] expectedOptionsKeys, [2]
expectedOptionsValues.
+ final List<TestTuple<Pair<Config, Boolean>, Object[]>> testCases = new
LinkedList<>();
+
+ // Create test cases for Executor/Manager on even/odd indices respectively.
+ for (int idx = 0; idx < 2; ++idx) {
+
+ // Manager case is default.
+ boolean isExecutor = false;
+ String key = keyManager;
+ String description = KubernetesConstants.MANAGER_NAME;
+
+ // Executor case.
+ if (idx % 2 == 0) {
+ isExecutor = true;
+ key = keyExecutor;
+ description = KubernetesConstants.EXECUTOR_NAME;
+ }
+
+ final String storageClassKeyOne = String.format(key, volumeNameOne,
storageClassField);
+ final String storageClassKeyTwo = String.format(key, volumeNameTwo,
storageClassField);
+ final String pathKeyOne = String.format(key, volumeNameOne, pathField);
+ final String pathKeyTwo = String.format(key, volumeNameTwo, pathField);
+ final String claimNameKeyOne = String.format(key, volumeNameOne,
claimNameField);
+ final String claimNameKeyTwo = String.format(key, volumeNameTwo,
claimNameField);
+
+ final Config configPVC = Config.newBuilder()
+ .put(pathKeyOne, expectedPath)
+ .put(pathKeyTwo, expectedPath)
+ .put(claimNameKeyOne, claimName)
+ .put(claimNameKeyTwo, claimName)
+ .put(storageClassKeyOne, expectedStorageClass)
+ .put(storageClassKeyTwo, expectedStorageClass)
+ .build();
+
+ final List<String> expectedKeys = Arrays.asList(volumeNameOne,
volumeNameTwo);
+ final List<VolumeClaimTemplateConfigKeys> expectedOptionsKeys =
+ Arrays.asList(VolumeClaimTemplateConfigKeys.path,
+ VolumeClaimTemplateConfigKeys.storageClassName,
+ VolumeClaimTemplateConfigKeys.claimName);
+ final List<String> expectedOptionsValues =
+ Arrays.asList(expectedPath, expectedStorageClass, claimName);
+
+ testCases.add(new TestTuple<>(description,
+ new Pair<>(configPVC, isExecutor),
+ new Object[]{expectedKeys, expectedOptionsKeys,
expectedOptionsValues}));
+ }
+
+ // Test loop.
+ for (TestTuple<Pair<Config, Boolean>, Object[]> testCase : testCases) {
+ final Map<String, Map<VolumeClaimTemplateConfigKeys, String>> mapOfPVC =
+ KubernetesContext.getVolumeClaimTemplates(testCase.input.first,
testCase.input.second);
+
+ Assert.assertTrue(testCase.description + ": Contains all provided
Volumes",
+ mapOfPVC.keySet().containsAll((List<String>) testCase.expected[0]));
+ for (Map<VolumeClaimTemplateConfigKeys, String> items :
mapOfPVC.values()) {
+ Assert.assertTrue(testCase.description + ": Contains all provided
option keys",
+ items.keySet().containsAll((List<VolumeClaimTemplateConfigKeys>)
testCase.expected[1]));
+ Assert.assertTrue(testCase.description + ": Contains all provided
option values",
+ items.values().containsAll((List<String>) testCase.expected[2]));
+ }
}
// Empty PVC.
- final Map<String, Map<VolumeClaimTemplateConfigKeys, String>> emptyPVC =
- KubernetesContext.getVolumeClaimTemplates(Config.newBuilder().build());
- Assert.assertTrue("Empty PVC is returned when no options provided",
emptyPVC.isEmpty());
+ final Boolean[] emptyPVCTestCases = new Boolean[] {true, false};
+ for (boolean testCase : emptyPVCTestCases) {
+ final Map<String, Map<VolumeClaimTemplateConfigKeys, String>> emptyPVC =
+
KubernetesContext.getVolumeClaimTemplates(Config.newBuilder().build(),
testCase);
+ Assert.assertTrue("Empty PVC is returned when no options provided",
emptyPVC.isEmpty());
+ }
}
@Test
@@ -146,8 +191,8 @@ public class KubernetesContextTest {
final String volumeNameInvalid = "volume-Name-Invalid";
final String failureValue = "Should-Fail";
final String generalFailureMessage = "Invalid Persistent Volume";
- final String keyPattern = KubernetesContext.KUBERNETES_VOLUME_CLAIM_PREFIX
- + "%s.%s";
+ final String keyPattern =
String.format(KubernetesContext.KUBERNETES_VOLUME_CLAIM_PREFIX
+ + "%%s.%%s", KubernetesConstants.EXECUTOR_NAME);
final List<TestTuple<Config, String>> testCases = new LinkedList<>();
// Invalid option key test.
@@ -186,11 +231,15 @@ public class KubernetesContextTest {
configInvalidStorageClassName, "Option `storageClassName`"));
// Testing loop.
+ final Boolean[] executorFlags = new Boolean[] {true, false};
for (TestTuple<Config, String> testCase : testCases) {
- try {
- KubernetesContext.getVolumeClaimTemplates(testCase.input);
- } catch (TopologySubmissionException e) {
- Assert.assertTrue(testCase.description,
e.getMessage().contains(testCase.expected));
+ // Test for both Executor and Manager.
+ for (boolean isExecutor : executorFlags) {
+ try {
+ KubernetesContext.getVolumeClaimTemplates(testCase.input,
isExecutor);
+ } catch (TopologySubmissionException e) {
+ Assert.assertTrue(testCase.description,
e.getMessage().contains(testCase.expected));
+ }
}
}
}
diff --git
a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/V1ControllerTest.java
b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/V1ControllerTest.java
index b543ea9..e79760d 100644
---
a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/V1ControllerTest.java
+++
b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/V1ControllerTest.java
@@ -103,132 +103,167 @@ public class V1ControllerTest {
+ " limits:\n"
+ " cpu: \"400m\"\n"
+ " memory: \"512M\"";
-
- private final Config config = Config.newBuilder().build();
- private final Config configWithPodTemplate = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
CONFIGMAP_POD_TEMPLATE_NAME)
+ private static final String POD_TEMPLATE_LOCATION_EXECUTOR =
+ String.format(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
+ KubernetesConstants.EXECUTOR_NAME);
+ private static final String POD_TEMPLATE_LOCATION_MANAGER =
+ String.format(KubernetesContext.KUBERNETES_POD_TEMPLATE_LOCATION,
+ KubernetesConstants.MANAGER_NAME);
+
+ private static final Config CONFIG = Config.newBuilder().build();
+ private static final Config CONFIG_WITH_POD_TEMPLATE = Config.newBuilder()
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR, CONFIGMAP_POD_TEMPLATE_NAME)
+ .put(POD_TEMPLATE_LOCATION_MANAGER, CONFIGMAP_POD_TEMPLATE_NAME)
.build();
- private final Config runtime = Config.newBuilder()
+ private static final Config RUNTIME = Config.newBuilder()
.put(Key.TOPOLOGY_NAME, TOPOLOGY_NAME)
.build();
private final Config configDisabledPodTemplate = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
CONFIGMAP_POD_TEMPLATE_NAME)
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_DISABLED,
"true")
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR, CONFIGMAP_POD_TEMPLATE_NAME)
+ .put(POD_TEMPLATE_LOCATION_MANAGER, CONFIGMAP_POD_TEMPLATE_NAME)
+ .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_DISABLED, "true")
.build();
@Spy
private final V1Controller v1ControllerWithPodTemplate =
- new V1Controller(configWithPodTemplate, runtime);
+ new V1Controller(CONFIG_WITH_POD_TEMPLATE, RUNTIME);
@Spy
private final V1Controller v1ControllerPodTemplate =
- new V1Controller(configDisabledPodTemplate, runtime);
+ new V1Controller(configDisabledPodTemplate, RUNTIME);
@Rule
public final ExpectedException expectedException = ExpectedException.none();
@Test
public void testLoadPodFromTemplateDefault() {
- final V1Controller v1ControllerNoPodTemplate = new V1Controller(config,
runtime);
- final V1PodTemplateSpec podSpec =
v1ControllerNoPodTemplate.loadPodFromTemplate();
+ final V1Controller v1ControllerNoPodTemplate = new V1Controller(CONFIG,
RUNTIME);
+ final V1PodTemplateSpec defaultPodSpec = new V1PodTemplateSpec();
+
+ final V1PodTemplateSpec podSpecExecutor =
v1ControllerNoPodTemplate.loadPodFromTemplate(true);
+ Assert.assertEquals("Default Pod Spec for Executor", defaultPodSpec,
podSpecExecutor);
- Assert.assertEquals(podSpec, new V1PodTemplateSpec());
+ final V1PodTemplateSpec podSpecManager =
v1ControllerNoPodTemplate.loadPodFromTemplate(false);
+ Assert.assertEquals("Default Pod Spec for Manager", defaultPodSpec,
podSpecManager);
}
@Test
public void testLoadPodFromTemplateNullConfigMap() {
- final String expected = "unable to locate";
- String message = "";
+ final List<TestTuple<Boolean, String>> testCases = new LinkedList<>();
+ testCases.add(new TestTuple<>("Executor not found", true, "unable to
locate"));
+ testCases.add(new TestTuple<>("Manager not found", false, "unable to
locate"));
- doReturn(null)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
+ for (TestTuple<Boolean, String> testCase : testCases) {
+ doReturn(null)
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ String message = "";
+ try {
+ v1ControllerWithPodTemplate.loadPodFromTemplate(testCase.input);
+ } catch (TopologySubmissionException e) {
+ message = e.getMessage();
+ }
+ Assert.assertTrue(testCase.description,
message.contains(testCase.expected));
}
- Assert.assertTrue(message.contains(expected));
}
@Test
public void testLoadPodFromTemplateNoConfigMap() {
- final String expected = "Failed to locate Pod Template";
- String message = "";
+ final List<TestTuple<Boolean, String>> testCases = new LinkedList<>();
+ testCases.add(new TestTuple<>("Executor no ConfigMap", true, "Failed to
locate Pod Template"));
+ testCases.add(new TestTuple<>("Manager no ConfigMap", false, "Failed to
locate Pod Template"));
- doReturn(new V1ConfigMap())
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
+ for (TestTuple<Boolean, String> testCase : testCases) {
+ doReturn(new V1ConfigMap())
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ String message = "";
+ try {
+ v1ControllerWithPodTemplate.loadPodFromTemplate(testCase.input);
+ } catch (TopologySubmissionException e) {
+ message = e.getMessage();
+ }
+ Assert.assertTrue(testCase.description,
message.contains(testCase.expected));
}
- Assert.assertTrue(message.contains(expected));
}
@Test
public void testLoadPodFromTemplateNoTargetConfigMap() {
- final String expected = "Failed to locate Pod Template";
- String message = "";
- V1ConfigMap configMapNoTargetData = new V1ConfigMapBuilder()
+ final List<TestTuple<Boolean, String>> testCases = new LinkedList<>();
+ testCases.add(new TestTuple<>("Executor no target ConfigMap",
+ true, "Failed to locate Pod Template"));
+ testCases.add(new TestTuple<>("Manager no target ConfigMap",
+ false, "Failed to locate Pod Template"));
+
+ final V1ConfigMap configMapNoTargetData = new V1ConfigMapBuilder()
.withNewMetadata()
.withName(CONFIGMAP_NAME)
.endMetadata()
.addToData("Dummy Key", "Dummy Value")
.build();
- doReturn(configMapNoTargetData)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
+ for (TestTuple<Boolean, String> testCase : testCases) {
+ doReturn(configMapNoTargetData)
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ String message = "";
+ try {
+ v1ControllerWithPodTemplate.loadPodFromTemplate(testCase.input);
+ } catch (TopologySubmissionException e) {
+ message = e.getMessage();
+ }
+ Assert.assertTrue(testCase.description,
message.contains(testCase.expected));
}
- Assert.assertTrue(message.contains(expected));
}
@Test
public void testLoadPodFromTemplateBadTargetConfigMap() {
- final String expected = "Error parsing";
- String message = "";
-
// ConfigMap with target ConfigMap and an invalid Pod Template.
- V1ConfigMap configMapInvalidPod = new V1ConfigMapBuilder()
+ final V1ConfigMap configMapInvalidPod = new V1ConfigMapBuilder()
.withNewMetadata()
.withName(CONFIGMAP_NAME)
.endMetadata()
.addToData(POD_TEMPLATE_NAME, "Dummy Value")
.build();
- doReturn(configMapInvalidPod)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
- }
- Assert.assertTrue("Invalid Pod Template parsing should fail",
message.contains(expected));
-
// ConfigMap with target ConfigMaps and an empty Pod Template.
- V1ConfigMap configMapEmptyPod = new V1ConfigMapBuilder()
+ final V1ConfigMap configMapEmptyPod = new V1ConfigMapBuilder()
.withNewMetadata()
- .withName(CONFIGMAP_NAME)
+ .withName(CONFIGMAP_NAME)
.endMetadata()
.addToData(POD_TEMPLATE_NAME, "")
.build();
- doReturn(configMapEmptyPod)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
+ // Test case container.
+ // Input: ConfigMap to setup mock V1Controller, Boolean flag for
executor/manager switch.
+ // Output: The expected error message.
+ final List<TestTuple<Pair<V1ConfigMap, Boolean>, String>> testCases = new
LinkedList<>();
+ testCases.add(new TestTuple<>("Executor invalid Pod Template",
+ new Pair<>(configMapInvalidPod, true), "Error parsing"));
+ testCases.add(new TestTuple<>("Manager invalid Pod Template",
+ new Pair<>(configMapInvalidPod, false), "Error parsing"));
+ testCases.add(new TestTuple<>("Executor empty Pod Template",
+ new Pair<>(configMapEmptyPod, true), "Error parsing"));
+ testCases.add(new TestTuple<>("Manager empty Pod Template",
+ new Pair<>(configMapEmptyPod, false), "Error parsing"));
+
+ // Test loop.
+ for (TestTuple<Pair<V1ConfigMap, Boolean>, String> testCase : testCases) {
+ doReturn(testCase.input.first)
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ String message = "";
+ try {
+ v1ControllerWithPodTemplate.loadPodFromTemplate(testCase.input.second);
+ } catch (TopologySubmissionException e) {
+ message = e.getMessage();
+ }
+ Assert.assertTrue(testCase.description,
message.contains(testCase.expected));
}
- Assert.assertTrue("Empty Pod Template parsing should fail",
message.contains(expected));
}
@Test
@@ -272,18 +307,32 @@ public class V1ControllerTest {
// ConfigMap with valid Pod Template.
- V1ConfigMap configMapValidPod = new V1ConfigMapBuilder()
+ final V1ConfigMap configMapValidPod = new V1ConfigMapBuilder()
.withNewMetadata()
.withName(CONFIGMAP_NAME)
.endMetadata()
.addToData(POD_TEMPLATE_NAME, POD_TEMPLATE_VALID)
.build();
- doReturn(configMapValidPod)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- V1PodTemplateSpec podTemplateSpec =
v1ControllerWithPodTemplate.loadPodFromTemplate();
- Assert.assertTrue(podTemplateSpec.toString().contains(expected));
+ // Test case container.
+ // Input: ConfigMap to setup mock V1Controller, Boolean flag for
executor/manager switch.
+ // Output: The expected Pod template as a string.
+ final List<TestTuple<Pair<V1ConfigMap, Boolean>, String>> testCases = new
LinkedList<>();
+ testCases.add(new TestTuple<>("Executor valid Pod Template",
+ new Pair<>(configMapValidPod, true), expected));
+ testCases.add(new TestTuple<>("Manager valid Pod Template",
+ new Pair<>(configMapValidPod, false), expected));
+
+ // Test loop.
+ for (TestTuple<Pair<V1ConfigMap, Boolean>, String> testCase : testCases) {
+ doReturn(testCase.input.first)
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ V1PodTemplateSpec podTemplateSpec =
v1ControllerWithPodTemplate.loadPodFromTemplate(true);
+
+
Assert.assertTrue(podTemplateSpec.toString().contains(testCase.expected));
+ }
}
@Test
@@ -300,24 +349,37 @@ public class V1ControllerTest {
+ " labels:\n"
+ " app: heron-tracker\n"
+ " spec:\n";
- V1ConfigMap configMap = new V1ConfigMapBuilder()
+ final V1ConfigMap configMap = new V1ConfigMapBuilder()
.withNewMetadata()
.withName(CONFIGMAP_NAME)
.endMetadata()
.addToData(POD_TEMPLATE_NAME, invalidPodTemplate)
.build();
- final String expected = "Error parsing";
- String message = "";
- doReturn(configMap)
- .when(v1ControllerWithPodTemplate)
- .getConfigMap(anyString());
- try {
- v1ControllerWithPodTemplate.loadPodFromTemplate();
- } catch (TopologySubmissionException e) {
- message = e.getMessage();
+
+ // Test case container.
+ // Input: ConfigMap to setup mock V1Controller, Boolean flag for
executor/manager switch.
+ // Output: The expected Pod template as a string.
+ final List<TestTuple<Pair<V1ConfigMap, Boolean>, String>> testCases = new
LinkedList<>();
+ testCases.add(new TestTuple<>("Executor invalid Pod Template",
+ new Pair<>(configMap, true), "Error parsing"));
+ testCases.add(new TestTuple<>("Manager invalid Pod Template",
+ new Pair<>(configMap, false), "Error parsing"));
+
+ // Test loop.
+ for (TestTuple<Pair<V1ConfigMap, Boolean>, String> testCase : testCases) {
+ doReturn(testCase.input.first)
+ .when(v1ControllerWithPodTemplate)
+ .getConfigMap(anyString());
+
+ String message = "";
+ try {
+ v1ControllerWithPodTemplate.loadPodFromTemplate(testCase.input.second);
+ } catch (TopologySubmissionException e) {
+ message = e.getMessage();
+ }
+ Assert.assertTrue(message.contains(testCase.expected));
}
- Assert.assertTrue(message.contains(expected));
}
@Test
@@ -336,7 +398,7 @@ public class V1ControllerTest {
.getConfigMap(anyString());
try {
- v1ControllerPodTemplate.loadPodFromTemplate();
+ v1ControllerPodTemplate.loadPodFromTemplate(true);
} catch (TopologySubmissionException e) {
message = e.getMessage();
}
@@ -346,45 +408,41 @@ public class V1ControllerTest {
@Test
public void testGetPodTemplateLocationPassing() {
final Config testConfig = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
CONFIGMAP_POD_TEMPLATE_NAME)
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR, CONFIGMAP_POD_TEMPLATE_NAME)
.build();
- final V1Controller v1Controller = new V1Controller(testConfig, runtime);
+ final V1Controller v1Controller = new V1Controller(testConfig, RUNTIME);
final Pair<String, String> expected = new Pair<>(CONFIGMAP_NAME,
POD_TEMPLATE_NAME);
- Pair<String, String> actual;
// Correct parsing
- actual = v1Controller.getPodTemplateLocation();
- Assert.assertEquals(actual, expected);
+ final Pair<String, String> actual =
v1Controller.getPodTemplateLocation(true);
+ Assert.assertEquals(expected, actual);
}
@Test
public void testGetPodTemplateLocationNoConfigMap() {
expectedException.expect(TopologySubmissionException.class);
final Config testConfig = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
- ".POD-TEMPLATE-NAME").build();
- V1Controller v1Controller = new V1Controller(testConfig, runtime);
- v1Controller.getPodTemplateLocation();
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR, ".POD-TEMPLATE-NAME").build();
+ V1Controller v1Controller = new V1Controller(testConfig, RUNTIME);
+ v1Controller.getPodTemplateLocation(true);
}
@Test
public void testGetPodTemplateLocationNoPodTemplate() {
expectedException.expect(TopologySubmissionException.class);
final Config testConfig = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
- "CONFIGMAP-NAME.").build();
- V1Controller v1Controller = new V1Controller(testConfig, runtime);
- v1Controller.getPodTemplateLocation();
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR, "CONFIGMAP-NAME.").build();
+ V1Controller v1Controller = new V1Controller(testConfig, RUNTIME);
+ v1Controller.getPodTemplateLocation(true);
}
@Test
public void testGetPodTemplateLocationNoDelimiter() {
expectedException.expect(TopologySubmissionException.class);
final Config testConfig = Config.newBuilder()
- .put(KubernetesContext.KUBERNETES_POD_TEMPLATE_CONFIGMAP_NAME,
- "CONFIGMAP-NAMEPOD-TEMPLATE-NAME").build();
- V1Controller v1Controller = new V1Controller(testConfig, runtime);
- v1Controller.getPodTemplateLocation();
+ .put(POD_TEMPLATE_LOCATION_EXECUTOR,
"CONFIGMAP-NAMEPOD-TEMPLATE-NAME").build();
+ V1Controller v1Controller = new V1Controller(testConfig, RUNTIME);
+ v1Controller.getPodTemplateLocation(true);
}
@Test
@@ -499,10 +557,12 @@ public class V1ControllerTest {
@Test
public void testConfigureContainerResources() {
+ final boolean isExecutor = true;
+
final Resource resourceDefault = new Resource(
- 9, ByteAmount.fromGigabytes(19), ByteAmount.fromGigabytes(99));
+ 9, ByteAmount.fromMegabytes(19000), ByteAmount.fromMegabytes(99000));
final Resource resourceCustom = new Resource(
- 4, ByteAmount.fromGigabytes(34), ByteAmount.fromGigabytes(400));
+ 4, ByteAmount.fromMegabytes(34000), ByteAmount.fromMegabytes(400000));
final Quantity defaultRAM = Quantity.fromString(
KubernetesUtils.Megabytes(resourceDefault.getRam()));
@@ -513,7 +573,7 @@ public class V1ControllerTest {
final Quantity customCPU = Quantity.fromString(
Double.toString(V1Controller.roundDecimal(resourceCustom.getCpu(),
3)));
final Quantity customDisk = Quantity.fromString(
-
Double.toString(V1Controller.roundDecimal(resourceCustom.getDisk().getValue(),
3)));
+ KubernetesUtils.Megabytes(resourceCustom.getDisk()));
final Config configNoLimit = Config.newBuilder()
.put(KubernetesContext.KUBERNETES_RESOURCE_REQUEST_MODE, "NOT_SET")
@@ -539,7 +599,7 @@ public class V1ControllerTest {
// Default. Null resources.
V1Container containerNull = new V1ContainerBuilder().build();
v1ControllerWithPodTemplate.configureContainerResources(
- containerNull, configNoLimit, resourceDefault);
+ containerNull, configNoLimit, resourceDefault, isExecutor);
Assert.assertTrue("Default LIMITS should be set in container with null
LIMITS",
containerNull.getResources().getLimits().entrySet()
.containsAll(expectDefaultRequirements.getLimits().entrySet()));
@@ -547,7 +607,7 @@ public class V1ControllerTest {
// Empty resources.
V1Container containerEmpty = new
V1ContainerBuilder().withNewResources().endResources().build();
v1ControllerWithPodTemplate.configureContainerResources(
- containerEmpty, configNoLimit, resourceDefault);
+ containerEmpty, configNoLimit, resourceDefault, isExecutor);
Assert.assertTrue("Default LIMITS should be set in container with empty
LIMITS",
containerNull.getResources().getLimits().entrySet()
.containsAll(expectDefaultRequirements.getLimits().entrySet()));
@@ -557,7 +617,7 @@ public class V1ControllerTest {
.withResources(customRequirements)
.build();
v1ControllerWithPodTemplate.configureContainerResources(
- containerCustom, configNoLimit, resourceDefault);
+ containerCustom, configNoLimit, resourceDefault, isExecutor);
Assert.assertTrue("Custom LIMITS should be set in container with custom
LIMITS",
containerCustom.getResources().getLimits().entrySet()
.containsAll(expectCustomRequirements.getLimits().entrySet()));
@@ -567,7 +627,7 @@ public class V1ControllerTest {
.withResources(customRequirements)
.build();
v1ControllerWithPodTemplate.configureContainerResources(
- containerRequests, configWithLimit, resourceDefault);
+ containerRequests, configWithLimit, resourceDefault, isExecutor);
Assert.assertTrue("Custom LIMITS should be set in container with custom
LIMITS and REQUEST",
containerRequests.getResources().getLimits().entrySet()
.containsAll(expectCustomRequirements.getLimits().entrySet()));
@@ -577,6 +637,49 @@ public class V1ControllerTest {
}
@Test
+ public void testConfigureContainerResourcesCLI() {
+ final boolean isExecutor = true;
+ final String customLimitMEMStr = "120Gi";
+ final String customLimitCPUStr = "5";
+ final String customRequestMEMStr = "100Mi";
+ final String customRequestCPUStr = "4";
+
+ final Resource resources = new Resource(
+ 6, ByteAmount.fromMegabytes(34000), ByteAmount.fromGigabytes(400));
+
+ final Quantity customLimitMEM = Quantity.fromString(customLimitMEMStr);
+ final Quantity customLimitCPU = Quantity.fromString(customLimitCPUStr);
+ final Quantity customRequestMEM = Quantity.fromString(customRequestMEMStr);
+ final Quantity customRequestCPU = Quantity.fromString(customRequestCPUStr);
+
+ final Config config = Config.newBuilder()
+ .put(String.format(KubernetesContext.KUBERNETES_RESOURCE_LIMITS_PREFIX
+ + KubernetesConstants.CPU, KubernetesConstants.EXECUTOR_NAME),
customLimitCPUStr)
+ .put(String.format(KubernetesContext.KUBERNETES_RESOURCE_LIMITS_PREFIX
+ + KubernetesConstants.MEMORY,
KubernetesConstants.EXECUTOR_NAME), customLimitMEMStr)
+
.put(String.format(KubernetesContext.KUBERNETES_RESOURCE_REQUESTS_PREFIX
+ + KubernetesConstants.CPU, KubernetesConstants.EXECUTOR_NAME),
customRequestCPUStr)
+
.put(String.format(KubernetesContext.KUBERNETES_RESOURCE_REQUESTS_PREFIX
+ + KubernetesConstants.MEMORY,
KubernetesConstants.EXECUTOR_NAME),
+ customRequestMEMStr)
+ .put(KubernetesContext.KUBERNETES_RESOURCE_REQUEST_MODE,
"EQUAL_TO_LIMIT")
+ .build();
+
+ final V1Container expected = new V1ContainerBuilder()
+ .withNewResources()
+ .addToLimits(KubernetesConstants.CPU, customLimitCPU)
+ .addToLimits(KubernetesConstants.MEMORY, customLimitMEM)
+ .addToRequests(KubernetesConstants.CPU, customRequestCPU)
+ .addToRequests(KubernetesConstants.MEMORY, customRequestMEM)
+ .endResources()
+ .build();
+
+ final V1Container actual = new V1Container();
+ v1ControllerWithPodTemplate.configureContainerResources(actual, config,
resources, isExecutor);
+ Assert.assertEquals("Container Resources are set from CLI.", expected,
actual);
+ }
+
+ @Test
public void testAddVolumesIfPresent() {
final String pathDefault = "config-host-volume-path";
final String pathNameDefault = "config-host-volume-name";
@@ -585,7 +688,7 @@ public class V1ControllerTest {
.put(KubernetesContext.KUBERNETES_VOLUME_TYPE, Volumes.HOST_PATH)
.put(KubernetesContext.KUBERNETES_VOLUME_HOSTPATH_PATH, pathDefault)
.build();
- final V1Controller controllerWithVol = new V1Controller(configWithVolumes,
runtime);
+ final V1Controller controllerWithVol = new V1Controller(configWithVolumes,
RUNTIME);
final V1Volume volumeDefault = new V1VolumeBuilder()
.withName(pathNameDefault)
@@ -613,7 +716,7 @@ public class V1ControllerTest {
final List<V1Volume> expectedCustom = Arrays.asList(volumeDefault,
volumeToBeKept);
// No Volumes set.
- V1Controller controllerDoNotSetVolumes = new
V1Controller(Config.newBuilder().build(), runtime);
+ V1Controller controllerDoNotSetVolumes = new
V1Controller(Config.newBuilder().build(), RUNTIME);
V1PodSpec podSpecNoSetVolumes = new V1PodSpec();
controllerDoNotSetVolumes.addVolumesIfPresent(podSpecNoSetVolumes);
Assert.assertNull(podSpecNoSetVolumes.getVolumes());
@@ -649,7 +752,7 @@ public class V1ControllerTest {
.put(KubernetesContext.KUBERNETES_CONTAINER_VOLUME_MOUNT_NAME,
pathNameDefault)
.put(KubernetesContext.KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH,
pathDefault)
.build();
- final V1Controller controllerWithMounts = new
V1Controller(configWithVolumes, runtime);
+ final V1Controller controllerWithMounts = new
V1Controller(configWithVolumes, RUNTIME);
final V1VolumeMount volumeDefault = new V1VolumeMountBuilder()
.withName(pathNameDefault)
.withMountPath(pathDefault)
@@ -670,7 +773,7 @@ public class V1ControllerTest {
);
// No Volume Mounts set.
- V1Controller controllerDoNotSetMounts = new
V1Controller(Config.newBuilder().build(), runtime);
+ V1Controller controllerDoNotSetMounts = new
V1Controller(Config.newBuilder().build(), RUNTIME);
V1Container containerNoSetMounts = new V1Container();
controllerDoNotSetMounts.mountVolumeIfPresent(containerNoSetMounts);
Assert.assertNull(containerNoSetMounts.getVolumeMounts());
@@ -1016,9 +1119,10 @@ public class V1ControllerTest {
.when(v1ControllerWithPodTemplate)
.createPersistentVolumeClaimVolumesAndMounts(anyMap());
+ // <configPVC> parameter is used in mock above, so we can set it to
<null> as it is not used.
v1ControllerWithPodTemplate
.configurePodWithPersistentVolumeClaimVolumesAndMounts((V1PodSpec)
testCase.input[0],
- (V1Container) testCase.input[1]);
+ (V1Container) testCase.input[1], null);
Assert.assertEquals("Pod Specs match " + testCase.description,
testCase.input[0], testCase.expected.first);
@@ -1026,4 +1130,96 @@ public class V1ControllerTest {
testCase.input[1], testCase.expected.second);
}
}
+
+ @Test
+ public void testSetShardIdEnvironmentVariableCommand() {
+
+ List<TestTuple<Boolean, String>> testCases = new LinkedList<>();
+
+ testCases.add(new TestTuple<>("Executor command is set correctly",
+ true, "SHARD_ID=$((${POD_NAME##*-} + 1)) && echo
shardId=${SHARD_ID}"));
+ testCases.add(new TestTuple<>("Manager command is set correctly",
+ false, "SHARD_ID=${POD_NAME##*-} && echo shardId=${SHARD_ID}"));
+
+ for (TestTuple<Boolean, String> testCase : testCases) {
+ Assert.assertEquals(testCase.description, testCase.expected,
+
v1ControllerWithPodTemplate.setShardIdEnvironmentVariableCommand(testCase.input));
+ }
+ }
+
+ @Test
+ public void testCreateResourcesRequirement() {
+ final String managerCpuLimit = "3000m";
+ final String managerMemLimit = "256Gi";
+ final Quantity memory = Quantity.fromString(managerMemLimit);
+ final Quantity cpu = Quantity.fromString(managerCpuLimit);
+ final List<TestTuple<Map<String, String>, Map<String, Quantity>>>
testCases =
+ new LinkedList<>();
+
+ // No input.
+ Map<String, String> inputEmpty = new HashMap<>();
+ testCases.add(new TestTuple<>("Empty input.", inputEmpty, new
HashMap<>()));
+
+ // Only memory.
+ Map<String, String> inputMemory = new HashMap<String, String>() {
+ {
+ put(KubernetesConstants.MEMORY, managerMemLimit);
+ }
+ };
+ Map<String, Quantity> expectedMemory = new HashMap<String, Quantity>() {
+ {
+ put(KubernetesConstants.MEMORY, memory);
+ }
+ };
+ testCases.add(new TestTuple<>("Only memory input.", inputMemory,
expectedMemory));
+
+ // Only CPU.
+ Map<String, String> inputCPU = new HashMap<String, String>() {
+ {
+ put(KubernetesConstants.CPU, managerCpuLimit);
+ }
+ };
+ Map<String, Quantity> expectedCPU = new HashMap<String, Quantity>() {
+ {
+ put(KubernetesConstants.CPU, cpu);
+ }
+ };
+ testCases.add(new TestTuple<>("Only CPU input.", inputCPU, expectedCPU));
+
+ // CPU and memory.
+ Map<String, String> inputMemoryCPU = new HashMap<String, String>() {
+ {
+ put(KubernetesConstants.MEMORY, managerMemLimit);
+ put(KubernetesConstants.CPU, managerCpuLimit);
+ }
+ };
+ Map<String, Quantity> expectedMemoryCPU = new HashMap<String, Quantity>() {
+ {
+ put(KubernetesConstants.MEMORY, memory);
+ put(KubernetesConstants.CPU, cpu);
+ }
+ };
+ testCases.add(new TestTuple<>("Memory and CPU input.", inputMemoryCPU,
expectedMemoryCPU));
+
+ // Invalid.
+ Map<String, String> inputInvalid = new HashMap<String, String>() {
+ {
+ put("invalid input", "will not be ignored");
+ put(KubernetesConstants.CPU, managerCpuLimit);
+ }
+ };
+ Map<String, Quantity> expectedInvalid = new HashMap<String, Quantity>() {
+ {
+ put(KubernetesConstants.CPU, cpu);
+ }
+ };
+ testCases.add(new TestTuple<>("Invalid input.", inputInvalid,
expectedInvalid));
+
+ // Test loop.
+ for (TestTuple<Map<String, String>, Map<String, Quantity>> testCase :
testCases) {
+ Map<String, Quantity> actual =
+ v1ControllerPodTemplate.createResourcesRequirement(testCase.input);
+ Assert.assertEquals(testCase.description, testCase.expected, actual);
+ }
+ }
}
diff --git a/website2/docs/schedulers-k8s-execution-environment.md
b/website2/docs/schedulers-k8s-execution-environment.md
new file mode 100644
index 0000000..cbc1956
--- /dev/null
+++ b/website2/docs/schedulers-k8s-execution-environment.md
@@ -0,0 +1,527 @@
+---
+id: schedulers-k8s-execution-environment
+title: Kubernetes Execution Environment Customization
+sidebar_label: Kubernetes Execution Environment Customization
+---
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Customizing the Heron Execution Environment
+
+This document demonstrates how you can customize various aspects of the Heron
execution environment when using the Kubernetes Scheduler.
+
+<br>
+
+***Table of contents:***
+- [Customizing the Heron Execution
Environment](#customizing-the-heron-execution-environment)
+ - [Customizing a Topology's Execution Environment Using Pod
Templates](#customizing-a-topologys-execution-environment-using-pod-templates)
+ - [Preparation](#preparation)
+ - [Pod Templates](#pod-templates)
+ - [Configuration Maps](#configuration-maps)
+ - [Submitting](#submitting)
+ - [Heron Configured Items in Pod
Templates](#heron-configured-items-in-pod-templates)
+ - [Executor and Manager Containers](#executor-and-manager-containers)
+ - [Pod](#pod)
+ - [Adding Persistent Volumes via the Command Line
Interface](#adding-persistent-volumes-via-the-command-line-interface)
+ - [Usage](#usage)
+ - [Example](#example)
+ - [Submitting](#submitting-1)
+ - [Required and Optional Configuration
Items](#required-and-optional-configuration-items)
+ - [Configuration Items Created and Entries
Made](#configuration-items-created-and-entries-made)
+ - [Setting Limits and Requests via the Command Line
Interface](#setting-limits-and-requests-via-the-command-line-interface)
+ - [Usage](#usage-1)
+ - [Example](#example-1)
+
+<br>
+
+---
+
+<br>
+
+## Customizing a Topology's Execution Environment Using Pod Templates
+
+<br>
+
+> This section demonstrates how you can utilize custom [Pod
Templates](https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates)
embedded in [Configuration
Maps](https://kubernetes.io/docs/concepts/configuration/configmap/) for your
Topology's `Executor`s and `Manager` (hereinafter referred to as `Heron
containers`). You may specify different Pod Templates for different topologies.
+
+<br/>
+
+When you deploy a topology to Heron on Kubernetes, you may specify individual
Pod Templates to be used in your topology's `Executor`s and `Manager`. This can
be achieved by providing valid Pod Templates, and embedding the Pod Templates
in Configuration Maps. By default, Heron will use a minimally configured Pod
Template which is adequate to deploy a topology.
+
+Pod Templates will allow you to configure most aspects of your topology's
execution environment, with some exceptions. There are some aspects of Pods for
which Heron will have the final say, and which will not be user-customizable.
Please view the [tables](#heron-configured-items-in-pod-templates) at the end
of this section to identify what is set by Heron.
+
+<br>
+
+> ***System Administrators:***
+>
+> * You may wish to disable the ability to load custom Pod Templates. To
achieve this, you must pass the define option `-D
heron.kubernetes.pod.template.disabled=true` to the Heron API Server on the
command line when launching. This command has been added to the Kubernetes
configuration files to deploy the Heron API Server and is set to `false` by
default.
+> * If you have a custom `Role` for the Heron API Server you will need to
ensure the `ServiceAccount` attached to the API server, via a `RoleBinding`,
has the correct permissions to access the `ConfigMaps`:
+>
+>```yaml
+>rules:
+>- apiGroups:
+> - ""
+> resources:
+> - configmaps
+> verbs:
+> - get
+> - list
+>```
+
+<br>
+
+### Preparation
+
+To deploy a custom Pod Template to Kubernetes with your topology, you must
provide a valid Pod Template embedded in a valid Configuration Map. We will be
using the following variables throughout this document, some of which are
reserved variable names:
+
+* `POD-TEMPLATE-NAME`: This is the name of the Pod Template's YAML definition
file. This is ***not*** a reserved variable and is a place-holder name.
+* `CONFIG-MAP-NAME`: This is the name that will be used by the Configuration
Map in which the Pod Template will be embedded by `kubectl`. This is ***not***
a reserved variable and is a place-holder name.
+* `heron.kubernetes.[executor | manager].pod.template`: This variable name is
used as the key passed to Heron for the `--config-property` on the CLI. This
***is*** a reserved variable name.
+
+***NOTE***: Please do ***not*** use the `.` (period character) in the name of
the `CONFIG-MAP-NAME`. This character will be used as a delimiter when
submitting your topologies.
+
+It is highly advised that you validate your Pod Templates before placing them
in a `ConfigMap` to isolate any validity issues using a tool such as
[Kubeval](https://kubeval.instrumenta.dev/) or the built-in `dry-run`
functionality in Kubernetes. Whilst these tools are handy, they will not catch
all potential errors in Kubernetes configurations.
+
+***NOTE***: When submitting a Pod Template to customize an `Executor` or
`Manager`, Heron will look for containers named `executor` and `manager`
respectively. These containers will be modified to support the functioning of
Heron, please read further below.
+
+#### Pod Templates
+
+An example of the Pod Template format is provided below, and is derived from
the configuration for the Heron Tracker Pod:
+
+```yaml
+apiVersion: v1
+kind: PodTemplate
+metadata:
+ name: heron-tracker
+ namespace: default
+template:
+ metadata:
+ labels:
+ app: heron-tracker
+ spec:
+ containers:
+ - name: heron-tracker
+ image: apache/heron:latest
+ ports:
+ - containerPort: 8888
+ name: api-port
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "200M"
+ limits:
+ cpu: "400m"
+ memory: "512M"
+```
+
+You would need to save this file as `POD-TEMPLATE-NAME`. Once you have a valid
Pod Template you may proceed to generate a `ConfigMap`.
+
+#### Configuration Maps
+
+> You must place the `ConfigMap` in the same namespace as the Heron API Server
using the `--namespace` option in the commands below if the API Server is not
in the `default` namespace.
+
+To generate a `ConfigMap` you will need to run the following command:
+
+```bash
+kubectl create configmap CONFIG-MAP-NAME --from-file path/to/POD-TEMPLATE-NAME
+```
+
+You may then want to verify the contents of the `ConfigMap` by running the
following command:
+
+```bash
+kubectl get configmaps CONFIG-MAP-NAME -o yaml
+```
+
+The `ConfigMap` should appear similar to the one below for our example:
+
+```yaml
+apiVersion: v1
+data:
+ POD-TEMPLATE-NAME: |
+ apiVersion: v1
+ kind: PodTemplate
+ metadata:
+ name: heron-tracker
+ namespace: default
+ template:
+ metadata:
+ labels:
+ app: heron-tracker
+ spec:
+ containers:
+ - name: heron-tracker
+ image: apache/heron:latest
+ ports:
+ - containerPort: 8888
+ name: api-port
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "200M"
+ limits:
+ cpu: "400m"
+ memory: "512M"
+kind: ConfigMap
+metadata:
+ creationTimestamp: "2021-09-27T21:55:30Z"
+ name: CONFIG-MAP-NAME
+ namespace: default
+ resourceVersion: "1313"
+ uid: ba001653-03d9-4ac8-804c-d2c55c974281
+```
+
+### Submitting
+
+To use the `ConfigMap` for a topology you would will need to submit with the
additional flag `--confg-property`. The `--config-property key=value` takes a
key-value pair:
+
+* Key: `heron.kubernetes.[executor | manager].pod.template`
+* Value: `CONFIG-MAP-NAME.POD-TEMPLATE-NAME`
+
+Please note that you must concatenate `CONFIG-MAP-NAME` and
`POD-TEMPLATE-NAME` with a **`.`** (period character).
+
+For example:
+
+```bash
+heron submit kubernetes \
+
--service-url=http://localhost:8001/api/v1/namespaces/default/services/heron-apiserver:9000/proxy
\
+ ~/.heron/examples/heron-api-examples.jar \
+ org.apache.heron.examples.api.AckingTopology acking \
+ --config-property
heron.kubernetes.executor.pod.template=CONFIG-MAP-NAME.POD-TEMPLATE-NAME \
+ --config-property
heron.kubernetes.manager.pod.template=CONFIG-MAP-NAME.POD-TEMPLATE-NAME
+```
+
+### Heron Configured Items in Pod Templates
+
+Heron will locate the containers named `executor` and/or `manager` in the Pod
Template and customize them as outlined below. All other containers within the
Pod Templates will remain unchanged.
+
+#### Executor and Manager Containers
+
+All metadata for the `Heron containers` will be overwritten by Heron. In some
other cases, values from the Pod Template for the `executor` and `manager` will
be overwritten by Heron as outlined below.
+
+| Name | Description | Policy |
+|---|---|---|
+| `image` | The `Heron container`'s image. | Overwritten by Heron using values
from the config.
+| `env` | Environment variables are made available within the container. The
`HOST` and `POD_NAME` keys are required by Heron and are thus reserved. |
Merged with Heron's values taking precedence. Deduplication is based on `name`.
+| `ports` | Port numbers opened within the container. Some of these port
numbers are required by Heron and are thus reserved. The reserved ports are
defined in Heron's constants as [`6001`-`6010`]. | Merged with Heron's values
taking precedence. Deduplication is based on the `containerPort` value.
+| `limits` <br> `requests` | Heron will attempt to load values for `cpu` and
`memory` from configs. | Heron's values take precedence over those in the Pod
Templates.
+| `volumeMounts` | These are the mount points within the `Heron container` for
the `volumes` available in the Pod. | Merged with Heron's values taking
precedence. Deduplication is based on the `name` value.
+| Annotation: `prometheus.io/scrape` | Flag to indicate whether Prometheus
logs can be scraped and is set to `true`. | Value is overridden by Heron. |
+| Annotation `prometheus.io/port` | Port address for Prometheus log scraping
and is set to `8080`. | Values are overridden by Heron.
+| Annotation: Pod | Pod's revision/version hash. | Automatically set.
+| Annotation: Service | Labels services can use to attach to the Pod. |
Automatically set.
+| Label: `app` | Name of the application launching the Pod and is set to
`Heron`. | Values are overridden by Heron.
+| Label: `topology`| The name of topology which was provided when submitting.
| User-defined and supplied on the CLI.
+
+#### Pod
+
+The following items will be set in the Pod Template's `spec` by Heron.
+
+| Name | Description | Policy |
+|---|---|---|
+`terminationGracePeriodSeconds` | Grace period to wait before shutting down
the Pod after a `SIGTERM` signal and is set to `0` seconds. | Values are
overridden by Heron.
+| `tolerations` | Attempts to schedule Pods with `taints` onto nodes hosting
Pods with matching `taints`. The entries below are included by default. <br>
Keys:<br>`node.kubernetes.io/not-ready` <br> `node.kubernetes.io/unreachable`
<br> Values (common):<br> `operator: Exists`<br> `effect: NoExecute`<br>
`tolerationSeconds: 10L` | Merged with Heron's values taking precedence.
Deduplication is based on the `key` value.
+| `containers` | Configurations for containers to be launched within the Pod.
| All containers, excluding the `Heron container`s, are loaded as-is.
+| `volumes` | Volumes to be made available to the entire Pod. | Merged with
Heron's values taking precedence. Deduplication is based on the `name` value.
+| `secretVolumes` | Secrets to be mounted as volumes within the Pod. | Loaded
from the Heron configs if present.
+
+<br>
+
+---
+<br>
+
+## Adding Persistent Volumes via the Command Line Interface
+
+<br>
+
+> This section demonstrates how you can utilize both static and dynamically
backed [Persistent Volume
Claims](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) in
the `Executor` and `Manager` containers (hereinafter referred to as `Heron
containers`). You will need to enable Dynamic Provisioning in your Kubernetes
cluster to proceed to use the dynamic provisioning functionality.
+
+<br/>
+
+It is possible to leverage Persistent Volumes with custom Pod Templates but
the Volumes you add will be shared between all `Executor` Pods in the topology
when customizing the `Executor`s.
+
+The CLI commands allow you to configure a Persistent Volume Claim (dynamically
or statically backed) which will be unique and isolated to each Pod and mounted
in a single `Heron container` when you submit your topology with a claim name
of `OnDemand`. Using any claim name other than on `OnDemand` will permit you to
configure a shared Persistent Volume without a custom Pod Template which will
be shared between all `Executor` Pods when customizing them. The CLI commands
override any config [...]
+
+Some use cases include process checkpointing, caching of results for later use
in the process, intermediate results which could prove useful in analysis
(ETL/ELT to a data lake or warehouse), as a source of data enrichment, etc.
+
+**Note:** Heron ***will*** remove any dynamically backed Persistent Volume
Claims it creates when a topology is terminated. Please be aware that Heron
uses the following `Labels` to locate the claims it has created:
+```yaml
+metadata:
+ labels:
+ topology: <topology-name>
+ onDemand: true
+```
+
+<br>
+
+> ***System Administrators:***
+>
+> * You may wish to disable the ability to configure Persistent Volume Claims
specified via the CLI. To achieve this, you must pass the define option `-D
heron.kubernetes.persistent.volume.claims.cli.disabled=true`to the Heron API
Server on the command line when launching. This command has been added to the
Kubernetes configuration files to deploy the Heron API Server and is set to
`false` by default.
+> * If you have a custom `Role`/`ClusterRole` for the Heron API Server you
will need to ensure the `ServiceAccount` attached to the API server has the
correct permissions to access the `Persistent Volume Claim`s:
+>
+>```yaml
+>rules:
+>- apiGroups:
+> - ""
+> resources:
+> - persistentvolumeclaims
+> verbs:
+> - create
+> - delete
+> - get
+> - list
+> - deletecollection
+>```
+
+<br>
+
+### Usage
+
+To configure a Persistent Volume Claim you must use the `--config-property`
option with the `heron.kubernetes.[executor |
manager].volumes.persistentVolumeClaim.` command prefix. Heron will not
validate your Persistent Volume Claim configurations, so please validate them
to ensure they are well-formed. All names must comply with the [*lowercase
RFC-1123*](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)
standard.
+
+The command pattern is as follows:
+`heron.kubernetes.[executor | manager].volumes.persistentVolumeClaim.[VOLUME
NAME].[OPTION]=[VALUE]`
+
+The currently supported CLI `options` are:
+
+* `claimName`
+* `storageClass`
+* `sizeLimit`
+* `accessModes`
+* `volumeMode`
+* `path`
+* `subPath`
+
+***Note:*** A `claimName` of `OnDemand` will create unique Volumes for each
`Heron container` as well as deploy a Persistent Volume Claim for each Volume.
Any other claim name will result in a shared Volume being created between all
Pods in the topology.
+
+***Note:*** The `accessModes` must be a comma-separated list of values
*without* any white space. Valid values can be found in the [Kubernetes
documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes).
+
+***Note:*** If a `storageClassName` is specified and there are no matching
Persistent Volumes then [dynamic
provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/)
must be enabled. Kubernetes will attempt to locate a Persistent Volume that
matches the `storageClassName` before it attempts to use dynamic provisioning.
If a `storageClassName` is not specified there must be [Persistent
Volumes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persi
[...]
+
+<br>
+
+#### Example
+
+A series of example commands to add `Persistent Volumes` to `Executor`s, and
the `YAML` entries they make in their respective configurations, are as follows.
+
+***Dynamic:***
+
+```bash
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.storageClassName=storage-class-name-of-choice
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
+```
+
+Generated `Persistent Volume Claim`:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ labels:
+ app: heron
+ onDemand: "true"
+ topology: <topology-name>
+ name: volumenameofchoice-<topology-name>-[Ordinal]
+spec:
+ accessModes:
+ - comma
+ - separated
+ - list
+ resources:
+ requests:
+ storage: 555Gi
+ storageClassName: storage-class-name-of-choice
+ volumeMode: volume-mode-of-choice
+```
+
+Pod Spec entries for `Volume`:
+
+```yaml
+volumes:
+ - name: volumenameofchoice
+ persistentVolumeClaim:
+ claimName: volumenameofchoice-<topology-name>-[Ordinal]
+```
+
+`Executor` container entries for `Volume Mounts`:
+
+```yaml
+volumeMounts:
+ - mountPath: /path/to/mount
+ subPath: /sub/path/to/mount
+ name: volumenameofchoice
+```
+
+<br>
+
+***Static:***
+
+```bash
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
+```
+
+Generated `Persistent Volume Claim`:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ labels:
+ app: heron
+ onDemand: "true"
+ topology: <topology-name>
+ name: volumenameofchoice-<topology-name>-[Ordinal]
+spec:
+ accessModes:
+ - comma
+ - separated
+ - list
+ resources:
+ requests:
+ storage: 555Gi
+ storageClassName: standard
+ volumeMode: volume-mode-of-choice
+```
+
+Pod Spec entries for `Volume`:
+
+```yaml
+volumes:
+ - name: volumenameofchoice
+ persistentVolumeClaim:
+ claimName: volumenameofchoice-<topology-name>-[Ordinal]
+```
+
+`Executor` container entries for `Volume Mounts`:
+
+```yaml
+volumeMounts:
+ - mountPath: /path/to/mount
+ subPath: /sub/path/to/mount
+ name: volumenameofchoice
+```
+
+<br>
+
+### Submitting
+
+A series of example commands to sumbit a topology using the *dynamic* example
CLI commands above:
+
+```bash
+heron submit kubernetes \
+
--service-url=http://localhost:8001/api/v1/namespaces/default/services/heron-apiserver:9000/proxy
\
+ ~/.heron/examples/heron-api-examples.jar \
+ org.apache.heron.examples.api.AckingTopology acking \
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.storageClassName=storage-class-name-of-choice
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
\
+--config-property
heron.kubernetes.executor.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
+```
+
+### Required and Optional Configuration Items
+
+The following table outlines CLI options which are either ***required*** (
✅ ), ***optional*** ( ❔ ), or ***not available*** ( ❌ )
depending on if you are using dynamically/statically backed or shared `Volume`s.
+
+| Option | Dynamic | Static | Shared
+|---|---|---|---|
+| `VOLUME NAME` | ✅ | ✅ | ✅
+| `claimName` | `OnDemand` | `OnDemand` | A valid name
+| `path` | ✅ | ✅ | ✅
+| `subPath` | ❔ | ❔ | ❔
+| `storageClassName` | ✅ | ❌ | ❌
+| `accessModes` | ✅ | ✅ | ❌
+| `sizeLimit` | ❔ | ❔ | ❌
+| `volumeMode` | ❔ | ❔ | ❌
+
+<br>
+
+***Note:*** The `VOLUME NAME` will be extracted from the CLI command and a
`claimName` is a always required.
+
+<br>
+
+### Configuration Items Created and Entries Made
+
+The configuration items and entries in the tables below will made in their
respective areas.
+
+A `Volume` and a `Volume Mount` will be created for each `volume name` which
you specify. Additionally, one `Persistent Volume Claim` will be created for
each `Volume` specified as dynamic using the `OnDemand` claim name.
+
+| Name | Description | Policy |
+|---|---|---|
+| `VOLUME NAME` | The `name` of the `Volume`. | Entries made in the
`Persistent Volume Claim`'s spec, the Pod Spec's `Volumes`, and the `Heron
containers` `volumeMounts`.
+| `claimName` | A Claim name for the Persistent Volume. | If `OnDemand` is
provided as the parameter then a unique Volume and Persistent Volume Claim will
be created. Any other name will result in a shared Volume between all Pods in
the topology with only a Volume and Volume Mount being added.
+| `path` | The `mountPath` of the `Volume`. | Entries made in the `Heron
containers` `volumeMounts`.
+| `subPath` | The `subPath` of the `Volume`. | Entries made in the `Heron
containers` `volumeMounts`.
+| `storageClassName` | The identifier name used to reference the dynamic
`StorageClass`. | Entries made in the `Persistent Volume Claim` and Pod Spec's
`Volume`.
+| `accessModes` | A comma-separated list of [access
modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes).
| Entries made in the `Persistent Volume Claim`.
+| `sizeLimit` | A resource request for storage space
[units](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory).
| Entries made in the `Persistent Volume Claim`.
+| `volumeMode` | Either `FileSystem` (default) or `Block` (raw block). [Read
more](https://kubernetes.io/docs/concepts/storage/_print/#volume-mode). |
Entries made in the `Persistent Volume Claim`.
+| Labels | Two labels for `topology` and `onDemand` provisioning are added. |
These labels are only added to dynamically backed `Persistent Volume Claim`s
created by Heron to support the removal of any claims created when a topology
is terminated.
+
+<br>
+
+---
+
+<br>
+
+## Setting Limits and Requests via the Command Line Interface
+
+> This section demonstrates how you can configure a topology's `Executor`
and/or `Manager` (hereinafter referred to as `Heron containers`) resource
`Requests` and `Limits` through CLI commands.
+
+<br/>
+
+You may configure an individual topology's `Heron container`'s resource
`Requests` and `Limits` during submission through CLI commands. The default
behaviour is to acquire values for resources from Configurations and for them
to be common between the `Executor`s and the `Manager` for a topology.
+
+<br>
+
+### Usage
+
+The command pattern is as follows:
+`heron.kubernetes.[executor | manager].[limits | requests].[OPTION]=[VALUE]`
+
+The currently supported CLI `options` and their associated `values` are:
+
+* `cpu`: A natural number indicating the number of [CPU
units](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-cpu).
+* `memory`: A natural number indicating the amount of [memory
units](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory).
+
+<br>
+
+#### Example
+
+An example submission command is as follows.
+
+***Limits and Requests:***
+
+```bash
+~/bin/heron submit kubernetes ~/.heron/examples/heron-api-examples.jar \
+org.apache.heron.examples.api.AckingTopology acking \
+--config-property heron.kubernetes.manager.limits.cpu=2 \
+--config-property heron.kubernetes.manager.limits.memory=3 \
+--config-property heron.kubernetes.manager.requests.cpu=1 \
+--config-property heron.kubernetes.manager.requests.memory=2
+```
diff --git a/website2/docs/schedulers-k8s-persistent-volume-claims.md
b/website2/docs/schedulers-k8s-persistent-volume-claims.md
deleted file mode 100644
index 994c22c..0000000
--- a/website2/docs/schedulers-k8s-persistent-volume-claims.md
+++ /dev/null
@@ -1,257 +0,0 @@
----
-id: schedulers-k8s-persistent-volume-claims
-title: Kubernetes Persistent Volume Claims via CLI
-sidebar_label: Kubernetes Persistent Volume Claims (CLI)
----
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-> This document demonstrates how you can utilize both static and dynamically
backed [Persistent Volume
Claims](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) in
the `Executor` containers. You will need to enable Dynamic Provisioning in your
Kubernetes cluster to proceed to use the dynamic provisioning functionality.
-
-<br/>
-
-It is possible to leverage Persistent Volumes with custom Pod Templates but
the Volumes you add will be shared between all Pods in the topology.
-
-The CLI commands allow you to configure a Persistent Volume Claim (dynamically
or statically backed) which will be unique and isolated to each Pod and mounted
in a single `Executor` when you submit your topology with a Claim name of
`OnDemand`. Using any Claim name other than on `OnDemand` will permit you to
configure a shared Persistent Volume without a custom Pod Template which will
be specific to an individual Pod. The CLI commands override any configurations
you may have present in t [...]
-
-Some use cases include process checkpointing, caching of results for later use
in the process, intermediate results which could prove useful in analysis
(ETL/ELT to a data lake or warehouse), as a source of data enrichment, etc.
-
-**Note:** Heron ***will*** remove any dynamically backed Persistent Volume
Claims it creates when a topology is terminated. Please be aware that Heron
uses the following `Labels` to locate the claims it has created:
-```yaml
-metadata:
- labels:
- topology: <topology-name>
- onDemand: true
-```
-
-<br>
-
-> ***System Administrators:***
->
-> * You may wish to disable the ability to configure dynamic Persistent Volume
Claims specified on the CLI. To achieve this, you must pass the define option
`-D heron.kubernetes.persistent.volume.claims.cli.disabled=true` to the Heron
API Server on the command line during boot. This command has been added to the
Kubernetes configuration files to deploy the Heron API Server and is set to
`false` by default.
-> * If you have a custom `Role`/`ClusterRole` for the Heron API Server you
will need to ensure the `ServiceAccount` attached to the API server has the
correct permissions to access the `Persistent Volume Claim`s:
->
->```yaml
->rules:
->- apiGroups:
-> - ""
-> resources:
-> - persistentvolumeclaims
-> verbs:
-> - create
-> - delete
-> - get
-> - list
-> - deletecollection
->```
-
-<br>
-
-## Usage
-
-To configure a Persistent Volume Claim you must use the `--config-property`
option with the `heron.kubernetes.volumes.persistentVolumeClaim.` command
prefix. Heron will not validate your Persistent Volume Claim configurations, so
please validate them to ensure they are well-formed. All names must comply with
the [*lowercase
RFC-1123*](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)
standard.
-
-The command pattern is as follows:
-`heron.kubernetes.volumes.persistentVolumeClaim.[VOLUME NAME].[OPTION]=[VALUE]`
-
-The currently supported CLI `options` are:
-
-* `claimName`
-* `storageClass`
-* `sizeLimit`
-* `accessModes`
-* `volumeMode`
-* `path`
-* `subPath`
-
-***Note:*** A `claimName` of `OnDemand` will create unique Volumes for each
`Executor` as well as deploy a Persistent Volume Claim for each Volume. Any
other Claim name will result in a shared Volume being created between all Pods
in the topology.
-
-***Note:*** The `accessModes` must be a comma separated list of values
*without* any white space. Valid values can be found in the [Kubernetes
documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes).
-
-***Note:*** If a `storageClassName` is specified and there are no matching
Persistent Volumes then [dynamic
provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/)
must be enabled. Kubernetes will attempt to locate a Persistent Volume that
matches the `storageClassName` before it attempts to use dynamic provisioning.
If a `storageClassName` is not specified there must be [Persistent
Volumes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persi
[...]
-
-<br>
-
-### Example
-
-An example series of commands and the `YAML` entries they make in their
respective configurations are as follows.
-
-***Dynamic:***
-
-```bash
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.storageClassName=storage-class-name-of-choice
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
-```
-
-Generated `Persistent Volume Claim`:
-
-```yaml
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- labels:
- app: heron
- onDemand: "true"
- topology: <topology-name>
- name: volumenameofchoice-<topology-name>-[Ordinal]
-spec:
- accessModes:
- - comma
- - separated
- - list
- resources:
- requests:
- storage: 555Gi
- storageClassName: storage-class-name-of-choice
- volumeMode: volume-mode-of-choice
-```
-
-Pod Spec entries for `Volume`:
-
-```yaml
-volumes:
- - name: volumenameofchoice
- persistentVolumeClaim:
- claimName: volumenameofchoice-<topology-name>-[Ordinal]
-```
-
-`Executor` container entries for `Volume Mounts`:
-
-```yaml
-volumeMounts:
- - mountPath: /path/to/mount
- subPath: /sub/path/to/mount
- name: volumenameofchoice
-```
-
-<br>
-
-***Static:***
-
-```bash
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
-```
-
-Generated `Persistent Volume Claim`:
-
-```yaml
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- labels:
- app: heron
- onDemand: "true"
- topology: <topology-name>
- name: volumenameofchoice-<topology-name>-[Ordinal]
-spec:
- accessModes:
- - comma
- - separated
- - list
- resources:
- requests:
- storage: 555Gi
- storageClassName: standard
- volumeMode: volume-mode-of-choice
-```
-
-Pod Spec entries for `Volume`:
-
-```yaml
-volumes:
- - name: volumenameofchoice
- persistentVolumeClaim:
- claimName: volumenameofchoice-<topology-name>-[Ordinal]
-```
-
-`Executor` container entries for `Volume Mounts`:
-
-```yaml
-volumeMounts:
- - mountPath: /path/to/mount
- subPath: /sub/path/to/mount
- name: volumenameofchoice
-```
-
-<br>
-
-## Submitting
-
-An example of sumbitting a topology using the *dynamic* example CLI commands
above:
-
-```bash
-heron submit kubernetes \
-
--service-url=http://localhost:8001/api/v1/namespaces/default/services/heron-apiserver:9000/proxy
\
- ~/.heron/examples/heron-api-examples.jar \
- org.apache.heron.examples.api.AckingTopology acking \
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.claimName=OnDemand
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.storageClassName=storage-class-name-of-choice
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.accessModes=comma,separated,list
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.sizeLimit=555Gi
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.volumeMode=volume-mode-of-choice
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.path=/path/to/mount
\
---config-property
heron.kubernetes.volumes.persistentVolumeClaim.volumenameofchoice.subPath=/sub/path/to/mount
-```
-
-## Required and Optional Configuration Items
-
-The following table outlines CLI options which are either ***required*** (
✅ ), ***optional*** ( ❔ ), or ***not available*** ( ❌ )
depending on if you are using dynamic/statically backed or shared `Volume`.
-
-| Option | Dynamic | Static | Shared
-|---|---|---|---|
-| `VOLUME NAME` | ✅ | ✅ | ✅
-| `claimName` | `OnDemand` | `OnDemand` | A valid name
-| `path` | ✅ | ✅ | ✅
-| `subPath` | ❔ | ❔ | ❔
-| `storageClassName` | ✅ | ❌ | ❌
-| `accessModes` | ✅ | ✅ | ❌
-| `sizeLimit` | ❔ | ❔ | ❌
-| `volumeMode` | ❔ | ❔ | ❌
-
-<br>
-
-***Note:*** The `VOLUME NAME` will be extracted from the CLI command and a
`claimName` is a always required.
-
-<br>
-
-## Configuration Items Created and Entries Made
-
-The configuration items and entries in the tables below will made in their
respective areas.
-
-One `Persistent Volume Claim`, a `Volume`, and a `Volume Mount` will be
created for each `volume name` which you specify. Each will be unique to a Pod
within the topology.
-
-| Name | Description | Policy |
-|---|---|---|
-| `VOLUME NAME` | The `name` of the `Volume`. | Entries made in the
`Persistent Volume Claim`'s spec, the Pod Spec's `Volumes`, and the `executor`
containers `volumeMounts`.
-| `claimName` | A Claim name for the Persistent Volume. | If `OnDemand` is
provided as the parameter then a unique Volume and Persistent Volume Claim will
be created. Any other name will result in a shared Volume between all Pods in
the topology with only a Volume and Volume Mount being added.
-| `path` | The `mountPath` of the `Volume`. | Entries made in the `executor`
containers `volumeMounts`.
-| `subPath` | The `subPath` of the `Volume`. | Entries made in the `executor`
containers `volumeMounts`.
-| `storageClassName` | The identifier name used to reference the dynamic
`StorageClass`. | Entries made in the `Persistent Volume Claim` and Pod Spec's
`Volume`.
-| `accessModes` | A comma separated list of access modes. | Entries made in
the `Persistent Volume Claim`.
-| `sizeLimit` | A resource request for storage space. | Entries made in the
`Persistent Volume Claim`.
-| `volumeMode` | Either `FileSystem` (default) or `Block` (raw block). [Read
more](https://kubernetes.io/docs/concepts/storage/_print/#volume-mode). |
Entries made in the `Persistent Volume Claim`.
-| Labels | Two labels for `topology` and `onDemand` provisioning are added. |
These labels are only added to dynamically backed `Persistent Volume Claim`'s
created by Heron to support the removal of any claims created when a topology
is terminated.
diff --git a/website2/docs/schedulers-k8s-pod-templates.md
b/website2/docs/schedulers-k8s-pod-templates.md
deleted file mode 100644
index e6fe3c9..0000000
--- a/website2/docs/schedulers-k8s-pod-templates.md
+++ /dev/null
@@ -1,201 +0,0 @@
----
-id: schedulers-k8s-pod-templates
-title: Kubernetes Pod Templates
-sidebar_label: Kubernetes Pod Templates
----
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-> This document demonstrates how you can utilize custom [Pod
Templates](https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates)
embedded in [Configuration
Maps](https://kubernetes.io/docs/concepts/configuration/configmap/) for your
computation nodes - i.e., Spouts and Bolts. You may specify different Pod
Templates for different topologies.
-
-<br/>
-
-When you deploy a topology to Heron on Kubernetes, you may specify a Pod
Template to be used on the computation nodes. This can be achieved by providing
a valid Pod Template, and embedding the Pod Template within a Configuration
Map. By default, Heron will use a minimally configured Pod Template which is
adequate to deploy a topology.
-
-Pod Templates will allow you to configure most aspects of the Pods where the
computations occur, with some exceptions. There are some aspects of Pods for
which Heron will have the final say, and which will not be user-customizable.
Please view the tables at the end of this document to identify what is set by
Heron.
-
-<br>
-
-> ***System Administrators:***
->
-> * You may wish to disable the ability to load custom Pod Templates. To
achieve this, you must pass the define option `-D
heron.kubernetes.pod.template.configmap.disabled=true` to the Heron API Server
on the command line during boot. This command has been added to the Kubernetes
configuration files to deploy the Heron API Server and is set to `false` by
default.
-> * If you have a custom `Role`/`ClusterRole` for the Heron API Server you
will need to ensure the `ServiceAccount` attached to the API server has the
correct permissions to access the `ConfigMaps`:
->
->```yaml
->rules:
->- apiGroups:
-> - ""
-> resources:
-> - configmaps
-> verbs:
-> - get
-> - watch
-> - list
->```
-
-<br>
-
-## Preparation
-
-To deploy a custom Pod Template to Kubernetes with your topology, you must
provide a valid Pod Template embedded in a valid Configuration Map. We will be
using the following variables throughout this document, some of which are
reserved variable names:
-
-* `POD-TEMPLATE-NAME`: This is the name of the Pod Template's YAML definition
file. This is ***not*** a reserved variable and is a place-holder name.
-* `CONFIG-MAP-NAME`: This is the name which will be used by the Configuration
Map in which the Pod Template will be embedded by `kubectl`. This is ***not***
a reserved variable and is a place-holder name.
-* `heron.kubernetes.pod.template.configmap.name`: This variable name used as
the key passed to Heron for the `--config-property` on the CLI. This ***is*** a
reserved variable name.
-
-***NOTE***: Please do ***not*** use the `.` (period character) in the name of
the `CONFIG-MAP-NAME`. This character will be used as a delimiter when
submitting your topologies.
-
-It is highly advised that you validate your Pod Templates before placing them
in a `ConfigMap` to isolate any validity issues using a tool such as
[Kubeval](https://kubeval.instrumenta.dev/).
-
-### Pod Templates
-
-An example of a Pod Template is provided below, and is derived from the
configuration for the Heron Tracker Pod:
-
-```yaml
-apiVersion: v1
-kind: PodTemplate
-metadata:
- name: heron-tracker
- namespace: default
-template:
- metadata:
- labels:
- app: heron-tracker
- spec:
- containers:
- - name: heron-tracker
- image: apache/heron:latest
- ports:
- - containerPort: 8888
- name: api-port
- resources:
- requests:
- cpu: "100m"
- memory: "200M"
- limits:
- cpu: "400m"
- memory: "512M"
-```
-
-You would need to save this file as `POD-TEMPLATE-NAME`. Once you have a valid
Pod Template you may proceed to generate a `ConfigMap`.
-
-### Configuration Maps
-
-> You must place the `ConfigMap` in the same namespace as the Heron API Server
using the `--namespace` option in the commands below if the server is not in
the `default` namespace.
-
-To generate a `ConfigMap` you will need to run the following command:
-
-```bash
-kubectl create configmap CONFIG-MAP-NAME --from-file path/to/POD-TEMPLATE-NAME
-```
-
-You may then want to verify the contents of the `ConfigMap` by running the
following command:
-
-```bash
-kubectl get configmaps CONFIG-MAP-NAME -o yaml
-```
-
-The `ConfigMap` should appear similar to the one below for our example:
-
-```yaml
-apiVersion: v1
-data:
- POD-TEMPLATE-NAME: |
- apiVersion: v1
- kind: PodTemplate
- metadata:
- name: heron-tracker
- namespace: default
- template:
- metadata:
- labels:
- app: heron-tracker
- spec:
- containers:
- - name: heron-tracker
- image: apache/heron:latest
- ports:
- - containerPort: 8888
- name: api-port
- resources:
- requests:
- cpu: "100m"
- memory: "200M"
- limits:
- cpu: "400m"
- memory: "512M"
-kind: ConfigMap
-metadata:
- creationTimestamp: "2021-09-27T21:55:30Z"
- name: CONFIG-MAP-NAME
- namespace: default
- resourceVersion: "1313"
- uid: ba001653-03d9-4ac8-804c-d2c55c974281
-```
-
-## Submitting
-
-To use the `ConfigMap` for a topology you would submit with the additional
flag `--confg-property`. The `--config-property key=value` takes a key value
pair:
-
-* Key: `heron.kubernetes.pod.template.configmap.name`
-* Value: `CONFIG-MAP-NAME.POD-TEMPLATE-NAME`
-
-Please note that you must concatenate `CONFIG-MAP-NAME` and
`POD-TEMPLATE-NAME` with a **`.`** (period chracter).
-
-For example:
-
-```bash
-heron submit kubernetes \
-
--service-url=http://localhost:8001/api/v1/namespaces/default/services/heron-apiserver:9000/proxy
\
- ~/.heron/examples/heron-api-examples.jar \
- org.apache.heron.examples.api.AckingTopology acking \
- --config-property
heron.kubernetes.pod.template.configmap.name=CONFIG-MAP-NAME.POD-TEMPLATE-NAME
-```
-
-## Heron Configured Items in Pod Templates
-
-Heron will locate the container named `executor` in the Pod Template and
customize it as outlined below. All other containers within the Pod Template
will remain unchanged.
-
-### Executor Container
-
-All metadata for the `executor` container will be overwritten by Heron. In
some other cases, values from the Pod Template for the `executor` will be
overwritten by Heron as outline below.
-
-| Name | Description | Policy |
-|---|---|---|
-| `image` | The `executor` container's image. | Overwritten by Heron using
values form the config.
-| `env` | Environment variables are made available within the container. The
`HOST` and `POD_NAME` keys are required by Heron and are thus reserved. |
Merged with Heron's values taking precedence. Deduplication is based on `name`.
-| `ports` | Port numbers opened within the container. Some of these port
number are required by Heron and are thus reserved. The reserved ports are
defined in Heron's constants as [`6001`-`6010`]. | Merged with Heron's values
taking precedence. Deduplication is based on the `containerPort` value.
-| `limits` | Heron will attempt to load values for `cpu` and `memory` from
configs. If these values are not provided in the Configs, then values from the
Pod Templates will be used. | Heron's values take precedence over those in the
Pod Templates.
-| `volumeMounts` | These are the mount points within the `executor` container
for the `volumes` available in the Pod. | Merged with Heron's values taking
precedence. Deduplication is based on the `name` value.
-| Annotation: `prometheus.io/scrape` | Flag to indicate whether Prometheus
logs can be scraped and is set to `true`. | Value is overridden by Heron. |
-| Annotation `prometheus.io/port` | Port address for Prometheus log scraping
and is set to `8080`. | Values are overridden by Heron.
-| Annotation: Pod | Pod's revision/version hash. | Automatically set.
-| Annotation: Service | Labels services can use to attach to the Pod. |
Automatically set.
-| Label: `app` | Name of the application lauching the Pod and is set to
`Heron`. | Values are overridden by Heron.
-| Label: `topology`| The name of topology which was provided when submitting.
| User defined and supplied on the CLI.
-
-### Pod
-
-The following items will be set in the Pod Template's `spec` by Heron.
-
-| Name | Description | Policy |
-|---|---|---|
-`terminationGracePeriodSeconds` | Grace period to wait before shutting down
the Pod after a `SIGTERM` signal and is set to `0` seconds. | Values are
overridden by Heron.
-| `tolerations` | Attempts to schedule Pods with `taints` onto nodes hosting
Pods with matching `taints`. The entries below are included by default. <br>
Keys:<br>`node.kubernetes.io/not-ready` <br> `node.kubernetes.io/unreachable`
<br> Values (common):<br> `operator: Exists`<br> `effect: NoExecute`<br>
`tolerationSeconds: 10L` | Merged with Heron's values taking precedence.
Deduplication is based on the `key` value.
-| `containers` | Configurations for containers to be launched within the Pod.
| All `containers`, excluding the `executor`, are loaded as-is.
-| `volumes` | Volumes to be made available to the entire Pod. | Merged with
Heron's values taking precedence. Deduplication is based on the `name` value.
-| `secretVolumes` | Secrets to be mounted as volumes within the Pod. | Loaded
from the Heron configs if present.
diff --git a/website2/website/sidebars.json b/website2/website/sidebars.json
index c803fdf..50c2086 100755
--- a/website2/website/sidebars.json
+++ b/website2/website/sidebars.json
@@ -53,8 +53,7 @@
"Schedulers": [
"schedulers-k8s-by-hand",
"schedulers-k8s-with-helm",
- "schedulers-k8s-pod-templates",
- "schedulers-k8s-persistent-volume-claims",
+ "schedulers-k8s-execution-environment",
"schedulers-aurora-cluster",
"schedulers-aurora-local",
"schedulers-local",
