hadoop git commit: Rebase onto latest trunk. minor conflicts
Repository: hadoop Updated Branches: refs/heads/yarn-native-services d924a4075 -> 5bbb430af Rebase onto latest trunk. minor conflicts Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bbb430a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bbb430a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bbb430a Branch: refs/heads/yarn-native-services Commit: 5bbb430af0d77dba7d946d0b815e4ed48c846e3e Parents: d924a40 Author: Jian HeAuthored: Wed Aug 30 22:48:35 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:49:20 2017 -0700 -- .../java/org/apache/hadoop/yarn/util/Apps.java | 2 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++-- 2 files changed, 4 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bbb430a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 75b4633..e88d2b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -150,6 +150,8 @@ public class Apps { } // Check if should black list the node based on container exit status + @Private + @Unstable public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { switch (exitStatus) { case ContainerExitStatus.PREEMPTED: http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bbb430a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 65412df..c2750c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BoundedAppender; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -1564,38 +1565,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { -switch (exitStatus) { -case ContainerExitStatus.PREEMPTED: -case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER: -case ContainerExitStatus.KILLED_BY_APPMASTER: -case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION: -case ContainerExitStatus.ABORTED: - // Neither the app's fault nor the system's fault. This happens by design, - // so no need for skipping nodes - return false; -case ContainerExitStatus.DISKS_FAILED: - // This container is marked with this exit-status means that the node is - // already marked as unhealthy given that most of the disks failed. So, no - // need for any explicit skipping of nodes. - return false; -case ContainerExitStatus.KILLED_EXCEEDED_VMEM: -case ContainerExitStatus.KILLED_EXCEEDED_PMEM: - // No point in skipping the node as it's not the system's fault - return false; -case ContainerExitStatus.SUCCESS: - return false; -case ContainerExitStatus.INVALID: - // Ideally, this shouldn't be considered for skipping a node. But in - // reality, it seems like there are cases where we are not setting - // exit-code correctly and so it's better to be conservative. See - // YARN-4284. - return true; -default: - return true; -} - } - private
[12/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java deleted file mode 100644 index da122da..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.servicemonitor; - -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.Map; - -/** - * Probe for a port being open. - */ -public class PortProbe extends Probe { - protected static final Logger log = LoggerFactory.getLogger(PortProbe.class); - private final int port; - private final int timeout; - - public PortProbe(int port, int timeout) { -super("Port probe of " + port + " for " + timeout + "ms", null); -this.port = port; -this.timeout = timeout; - } - - public static PortProbe create(Mapprops) - throws IOException { -int port = getPropertyInt(props, PORT_PROBE_PORT, null); - -if (port >= 65536) { - throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " + - "range"); -} - -int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT, -PORT_PROBE_CONNECT_TIMEOUT_DEFAULT); - -return new PortProbe(port, timeout); - } - - /** - * Try to connect to the (host,port); a failure to connect within - * the specified timeout is a failure. - * @param instance role instance - * @return the outcome - */ - @Override - public ProbeStatus ping(ComponentInstance instance) { -ProbeStatus status = new ProbeStatus(); - -if (instance.getContainerStatus() == null || SliderUtils -.isEmpty(instance.getContainerStatus().getIPs())) { - status.fail(this, new IOException( - instance.getCompInstanceName() + ": IP is not available yet")); - return status; -} - -String ip = instance.getContainerStatus().getIPs().get(0); -InetSocketAddress sockAddr = new InetSocketAddress(ip, port); -Socket socket = new Socket(); -try { - if (log.isDebugEnabled()) { -log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr -.toString() + ", timeout=" + MonitorUtils -.millisToHumanTime(timeout)); - } - socket.connect(sockAddr, timeout); - status.succeed(this); -} catch (Throwable e) { - String error = - instance.getCompInstanceName() + ": Probe " + sockAddr + " failed"; - log.debug(error, e); - status.fail(this, new IOException(error, e)); -} finally { - IOUtils.closeSocket(socket); -} -return status; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
[34/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java deleted file mode 100644 index 2f71004..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.api; - -/** - * These are the keys valid in resource options - * - /* - - Container failure window. - - The window is calculated in minutes as as (days * 24 *60 + hours* 24 + minutes) - - Every interval of this period after the AM is started/restarted becomes - the time period in which the CONTAINER_FAILURE_THRESHOLD value is calculated. - - After the window limit is reached, the failure counts are reset. This - is not a sliding window/moving average policy, simply a rule such as - "every six hours the failure count is reset" - - - - === - - - */ -public interface ResourceKeys { - - - /** - * #of instances of a component: {@value} - * - */ - String COMPONENT_INSTANCES = "yarn.component.instances"; - - /** - * Whether to use unique names for each instance of a component: {@value} - */ - String UNIQUE_NAMES = "component.unique.names"; - - /** - * Amount of memory to ask YARN for in MB. - * Important: this may be a hard limit on the - * amount of RAM that the service can use - * {@value} - */ - String YARN_MEMORY = "yarn.memory"; - - /** {@value} */ - int DEF_YARN_MEMORY = 256; - - /** - * Number of cores/virtual cores to ask YARN for - * {@value} - */ - String YARN_CORES = "yarn.vcores"; - - /** - * If normalization is set to false, then if the resource (memory and/or - * vcore) requested by a role is higher than YARN limits, then the resource - * request is not normalized. If this causes failures at the YARN level then - * applications are expecting that to happen. Default value is true. - */ - String YARN_RESOURCE_NORMALIZATION_ENABLED = - "yarn.resource.normalization.enabled"; - - /** - * Number of disks per instance to ask YARN for - * {@value} - */ - String YARN_DISKS = "yarn.disks.count-per-instance"; - - /** - * Disk size per disk to ask YARN for - * {@value} - */ - String YARN_DISK_SIZE = "yarn.disk.size"; - - /** {@value} */ - int DEF_YARN_CORES = 1; - - - /** - * Label expression that this container must satisfy - * {@value} - */ - String YARN_LABEL_EXPRESSION = "yarn.label.expression"; - - /** default label expression: */ - String DEF_YARN_LABEL_EXPRESSION = null; - - - /** - * Constant to indicate that the requirements of a YARN resource limit - * (cores, memory, ...) should be set to the maximum allowed by - * the queue into which the YARN container requests are placed. - */ - String YARN_RESOURCE_MAX = "max"; - - /** - * Mandatory property for all roles - * 1. this must be defined. - * 2. this must be >= 1 - * 3. this must not match any other role priority in the cluster. - */ - String COMPONENT_PRIORITY = "yarn.role.priority"; - - /** - * placement policy - */ - String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy"; - - /** - * Maximum number of node failures that can be tolerated by a component on a specific node - */ - String NODE_FAILURE_THRESHOLD = - "yarn.node.failure.threshold"; - - /** - * maximum number of failed containers (in a single role) - * before the cluster is deemed to have failed {@value} - */ - String CONTAINER_FAILURE_THRESHOLD = -
[21/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java deleted file mode 100644 index 06dde67..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ /dev/null @@ -1,2138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.health.HealthCheckRegistry; -import com.google.common.base.Preconditions; -import com.google.protobuf.BlockingService; -import org.apache.commons.collections.CollectionUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.service.ServiceOperations; -import org.apache.hadoop.service.ServiceStateChangeListener; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.TimelineV2Client; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; -import org.apache.hadoop.yarn.client.api.async.NMClientAsync; -import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; -import
[74/74] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang
YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d924a407 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d924a407 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d924a407 Branch: refs/heads/yarn-native-services Commit: d924a40751d2899db5cc6177de0c15bf30f3547a Parents: 0344e8a Author: Jian HeAuthored: Tue Aug 29 20:37:38 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:22:31 2017 -0700 -- .../java/org/apache/hadoop/yarn/service/api/records/Resource.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d924a407/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java index cec9de9..dfdf92a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -103,6 +104,7 @@ public class Resource extends BaseResource implements Cloneable { this.memory = memory; } + @JsonIgnore public long getMemoryMB() { if (this.memory == null) { return 0; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[68/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java new file mode 100644 index 000..fb2fd16 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -0,0 +1,654 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.UpdatedContainer; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.client.api.async.NMClientAsync; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.service.api.ServiceApiConstants; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType; +import org.apache.hadoop.yarn.service.component.Component; +import org.apache.hadoop.yarn.service.component.ComponentEvent; +import org.apache.hadoop.yarn.service.component.ComponentEventType; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; +import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; +import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; +import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; +import
[07/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java deleted file mode 100644 index 395ff22..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockFactory; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY; -import static org.apache.slider.server.appmaster.model.mock.MockFactory.AAROLE_2; - -/** - * Test Anti-affine placement. - */ -public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest -implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateAAPlacement.class); - - private static final int NODES = 3; - - /** - * The YARN engine has a cluster with very few nodes (3) and lots of - * containers, so if AA placement isn't working, there will be affine - * placements surfacing. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { -return new MockYarnEngine(NODES, 8); - } - - /** - * This is the simplest AA allocation: no labels, so allocate anywhere. - * @throws Throwable - */ - //@Test - public void testAllocateAANoLabel() throws Throwable { -RoleStatus aaRole = getAaRole(); - -assertTrue(cloneNodemap().size() > 0); - -// want multiple instances, so there will be iterations -aaRole.setDesired(2); - -List ops = appState.reviewRequestAndReleaseNodes(); -AMRMClient.ContainerRequest request = getSingleRequest(ops); -assertFalse(request.getRelaxLocality()); -assertEquals(request.getNodes().size(), engine.getCluster() -.getClusterSize()); -assertNull(request.getRacks()); -assertNotNull(request.getCapability()); - -Container allocated = engine.allocateContainer(request); - -// notify the container ane expect -List assignments = new ArrayList<>(); -List operations = new ArrayList<>(); -
[17/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java deleted file mode 100644 index ba923bc..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ /dev/null @@ -1,2120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.state; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.metrics2.lib.MutableGaugeInt; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ServiceApiConstants; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.api.proto.Messages; -import org.apache.slider.api.proto.Messages.ComponentCountProto; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.ApplicationState; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.api.types.ComponentInformation; -import org.apache.slider.api.types.RoleStatistics; -import org.apache.hadoop.yarn.service.conf.SliderExitCodes; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.NoSuchNodeException; -import org.apache.slider.core.exceptions.SliderInternalStateException; -import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; -import org.apache.slider.server.appmaster.management.MetricsConstants; -import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; -import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
[49/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java deleted file mode 100644 index b3fdef9..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.rest; - -import com.google.common.base.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -/** - * Class to bond to a Jersey client, for UGI integration and SPNEGO. - * - * Usage: create an instance, then when creating a Jersey Client - * pass in to the constructor the handler provided by {@link #getHandler()} - * - * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey docs - */ -public class UgiJerseyBinding implements -HttpURLConnectionFactory { - private static final Logger log = - LoggerFactory.getLogger(UgiJerseyBinding.class); - - private final UrlConnectionOperations operations; - private final URLConnectionClientHandler handler; - - /** - * Construct an instance - * @param operations operations instance - */ - @SuppressWarnings("ThisEscapedInObjectConstruction") - public UgiJerseyBinding(UrlConnectionOperations operations) { -Preconditions.checkArgument(operations != null, "Null operations"); -this.operations = operations; -handler = new URLConnectionClientHandler(this); - } - - /** - * Create an instance off the configuration. The SPNEGO policy - * is derived from the current UGI settings. - * @param conf config - */ - public UgiJerseyBinding(Configuration conf) { -this(new UrlConnectionOperations(conf)); - } - - /** - * Get a URL connection. - * @param url URL to connect to - * @return the connection - * @throws IOException any problem. {@link AuthenticationException} - * errors are wrapped - */ - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { -try { - // open a connection handling status codes and so redirections - // but as it opens a connection, it's less useful than you think. - - return operations.openConnection(url); -} catch (AuthenticationException e) { - throw new IOException(e); -} - } - - public UrlConnectionOperations getOperations() { -return operations; - } - - public URLConnectionClientHandler getHandler() { -return handler; - } - - /** - * Get the SPNEGO flag (as found in the operations instance - * @return the spnego policy - */ - public boolean isUseSpnego() { -return operations.isUseSpnego(); - } - - - /** - * Uprate error codes 400 and up into faults; - * - * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)} - */ - public static IOException uprateFaults(HttpVerb verb, String url, -
[48/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java deleted file mode 100644 index fe8cce8..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.StringWriter; -import java.net.URL; -import java.util.Map; - -/** - * Methods to aid in config, both in the Configuration class and - * with other parts of setting up Slider-initated processes. - * - * Some of the methods take an argument of a map iterable for their sources; this allows - * the same method - */ -public class ConfigHelper { - private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Mapmap, - String origin) throws BadConfigException { -addConfigMap(config, map.entrySet(), origin); - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Iterable > map, - String origin) throws BadConfigException { -for (Map.Entry mapEntry : map) { - String key = mapEntry.getKey(); - String value = mapEntry.getValue(); - if (value == null) { -throw new BadConfigException("Null value for property " + key); - } - config.set(key, value, origin); -} - } - - /** - * Convert to an XML string - * @param conf configuration - * @return conf - * @throws IOException - */ - public static String toXml(Configuration conf) throws IOException { -StringWriter writer = new StringWriter(); -conf.writeXml(writer); -return writer.toString(); - } - - - /** - * Register a resource as a default resource. - * Do not attempt to use this unless you understand that the - * order in which default resources are loaded affects the outcome, - * and that subclasses of Configuration often register new default - * resources - * @param resource the resource name - * @return the URL or null - */ - public static URL registerDefaultResource(String resource) { -URL resURL = getResourceUrl(resource); -if (resURL != null) { - Configuration.addDefaultResource(resource); -} -return resURL; - } - - /** - * Load a configuration from a resource on this classpath. - * If the resource is not found, an empty configuration is returned - * @param resource the resource name - * @return the loaded configuration. - */ - public static Configuration loadFromResource(String resource) { -Configuration conf = new Configuration(false); -URL resURL = getResourceUrl(resource); -if (resURL != null) { - log.debug("loaded resources from {}", resURL); - conf.addResource(resource); -} else{ - log.debug("failed to find {} on
[23/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java deleted file mode 100644 index bf71861..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.restclient; - -import com.google.common.base.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.slider.core.exceptions.ExceptionConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -/** - * Class to bond to a Jersey client, for UGI integration and SPNEGO. - * - * Usage: create an instance, then when creating a Jersey Client - * pass in to the constructor the handler provided by {@link #getHandler()} - * - * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey docs - */ -public class UgiJerseyBinding implements -HttpURLConnectionFactory { - private static final Logger log = - LoggerFactory.getLogger(UgiJerseyBinding.class); - - private final UrlConnectionOperations operations; - private final URLConnectionClientHandler handler; - - /** - * Construct an instance - * @param operations operations instance - */ - @SuppressWarnings("ThisEscapedInObjectConstruction") - public UgiJerseyBinding(UrlConnectionOperations operations) { -Preconditions.checkArgument(operations != null, "Null operations"); -this.operations = operations; -handler = new URLConnectionClientHandler(this); - } - - /** - * Create an instance off the configuration. The SPNEGO policy - * is derived from the current UGI settings. - * @param conf config - */ - public UgiJerseyBinding(Configuration conf) { -this(new UrlConnectionOperations(conf)); - } - - /** - * Get a URL connection. - * @param url URL to connect to - * @return the connection - * @throws IOException any problem. {@link AuthenticationException} - * errors are wrapped - */ - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { -try { - // open a connection handling status codes and so redirections - // but as it opens a connection, it's less useful than you think. - - return operations.openConnection(url); -} catch (AuthenticationException e) { - throw new IOException(e); -} - } - - public UrlConnectionOperations getOperations() { -return operations; - } - - public URLConnectionClientHandler getHandler() { -return handler; - } - - /** - * Get the SPNEGO flag (as found in the operations instance - * @return the spnego policy - */ - public boolean isUseSpnego() { -return operations.isUseSpnego(); - } - - - /** - * Uprate error codes 400 and up into faults; - * - * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)} - */ - public static
[71/74] [abbrv] hadoop git commit: YARN-5244. Documentation required for DNS Server implementation. Contributed by Jon Maron
YARN-5244. Documentation required for DNS Server implementation. Contributed by Jon Maron Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8673f24a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8673f24a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8673f24a Branch: refs/heads/yarn-native-services Commit: 8673f24a4a3f2288f284b3f14713a76ac90f552e Parents: 2b3aff2 Author: Gour SahaAuthored: Thu Aug 24 19:05:13 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:22:30 2017 -0700 -- hadoop-project/src/site/site.xml| 7 +- .../native-services/NativeServicesDiscovery.md | 127 +++ .../native-services/NativeServicesIntro.md | 0 .../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes .../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes .../resources/images/dns_record_removal.jpeg| Bin 0 -> 58041 bytes 6 files changed, 133 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8673f24a/hadoop-project/src/site/site.xml -- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index e7af227..837ac3f 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -148,7 +148,12 @@ - + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/8673f24a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md new file mode 100644 index 000..4a048af --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md @@ -0,0 +1,127 @@ +# YARN DNS Server +## Introduction + +The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions: + +1. **Exposing existing service discovery information via DNS**â - Information provided in +the current YARN service registryâs records will be converted into DNS entries, thus +allowing users to discover information about YARN applications using standard DNS +client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port +number for services). +2. **Enabling Container to IP mappingsâÂ** - Enables discovery of the IPs of containers via +standard DNS lookups. Given the availability of the records via DNS, container +nameÂbased communication will be facilitated (e.g. âcurl +http://myContainer.myDomain.com/endpointâ). + +## Service Properties + +The existing YARN Service Registry is leveraged as the source of information for the DNS Service. + +The following core functions are supported by the DNS Server: + +###Functional properties + +1. Supports creation of DNS records for endÂpoints of the deployed YARN applications +2. Record names remain unchanged during restart of containers and/or applications +3. Supports reverse lookups (name based on IP). +4. Supports security using the standards defined by The Domain Name System Security +Extensions (DNSSEC) +5. Highly available +6. Scalable Â- The service provides the responsiveness (e.g. low latency) required to +respond to DNS queries (timeouts yield attempts to invoke other configured name +servers). + +###Deployment properties + +1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as +a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a +primary DNS server and does not forward requests to other servers. +2. The DNS Server exposes a port that can receive both TCP and UDP requests per +DNS standards. The default port for DNS protocols is in a restricted, administrative port +range (53), so the port is configurable for deployments in which the service may +not be managed via an administrative account. + +##DNS Record Name Structure + +The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see âPreferred Name Syntaxâ in RFC 1035): + +* **domain** -â the name of the cluster DNS domain. This name is provided as a +configuration property. In addition, it is this name that is configured at a parent DNS +server as the
[72/74] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344e8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java index 6e6f4dd..7e53d18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java @@ -24,35 +24,17 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.GlobFilter; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.client.params.Arguments; import org.apache.hadoop.yarn.service.client.params.SliderActions; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; -import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.apache.zookeeper.server.util.KerberosUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,32 +45,19 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.io.Serializable; -import java.net.InetSocketAddress; import java.net.ServerSocket; -import java.net.Socket; import java.net.URL; import java.net.URLDecoder; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; /** * These are slider-specific Util methods @@ -97,41 +66,6 @@ public final class SliderUtils { private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); - /** - * Atomic bool to track whether or not process security has already been - * turned on (prevents re-entrancy) - */ - private static final AtomicBoolean processSecurityAlreadyInitialized = - new AtomicBoolean(false); - public static final String JAVA_SECURITY_KRB5_REALM = - "java.security.krb5.realm"; - public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc"; - - /** - * Winutils - */ - public static final String WINUTILS = "WINUTILS.EXE"; - /** - * name of openssl program - */ - public static final String OPENSSL = "openssl"; - - /** - * name of python program - */ - public static final String PYTHON = "python"; - - /** - * type of docker standalone service - */ - public static final String DOCKER = "docker"; - /** - * type of docker on yarn service - */ - public static final String DOCKER_YARN = "yarn_docker"; -
[70/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e94f23ee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e94f23ee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e94f23ee Branch: refs/heads/yarn-native-services Commit: e94f23ee15b06f82c9178f0db15522e29e3c9d7b Parents: 8673f24 Author: Billie RinaldiAuthored: Mon Aug 28 09:59:55 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:22:30 2017 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml |8 +- .../assemblies/hadoop-yarn-services-dist.xml|2 +- hadoop-project/pom.xml |2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn| 14 +- .../hadoop-yarn/conf/yarn-env.sh|2 +- .../hadoop-yarn-services-api/pom.xml|2 +- .../hadoop/yarn/service/webapp/ApiServer.java | 274 +++ .../yarn/service/webapp/ApiServerWebApp.java| 159 ++ .../service/webapp/ApplicationApiService.java | 275 --- .../service/webapp/ApplicationApiWebApp.java| 123 -- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 185 +- .../src/main/scripts/run_rest_service.sh| 28 - .../dev-support/findbugs-exclude.xml| 48 + .../conf/yarnservice-log4j.properties | 62 + .../hadoop-yarn-services-core/pom.xml | 408 + .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java| 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 156 ++ .../hadoop/yarn/service/ServiceMetrics.java | 98 + .../hadoop/yarn/service/ServiceScheduler.java | 654 +++ .../yarn/service/api/ServiceApiConstants.java | 69 + .../yarn/service/api/records/Artifact.java | 160 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 412 + .../yarn/service/api/records/ConfigFile.java| 225 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 297 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java| 102 ++ .../service/api/records/ReadinessCheck.java | 175 ++ .../yarn/service/api/records/Resource.java | 159 ++ .../yarn/service/api/records/Service.java | 466 + .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../hadoop/yarn/service/client/ServiceCLI.java | 112 ++ .../yarn/service/client/ServiceClient.java | 892 + .../client/params/AbstractActionArgs.java | 158 ++ .../client/params/AbstractArgsDelegate.java | 28 + .../AbstractClusterBuildingActionArgs.java | 58 + .../service/client/params/ActionBuildArgs.java | 31 + .../service/client/params/ActionClientArgs.java | 71 + .../service/client/params/ActionCreateArgs.java | 33 + .../client/params/ActionDependencyArgs.java | 65 + .../client/params/ActionDestroyArgs.java| 37 + .../service/client/params/ActionExistsArgs.java | 49 + .../service/client/params/ActionFlexArgs.java | 50 + .../service/client/params/ActionFreezeArgs.java | 56 + .../service/client/params/ActionHelpArgs.java | 44 + .../service/client/params/ActionKDiagArgs.java | 76 + .../service/client/params/ActionKeytabArgs.java | 76 + .../service/client/params/ActionListArgs.java | 76 + .../client/params/ActionRegistryArgs.java | 218 +++ .../client/params/ActionResolveArgs.java| 153 ++ .../client/params/ActionResourceArgs.java | 70 + .../service/client/params/ActionStatusArgs.java | 51 + .../service/client/params/ActionThawArgs.java | 67 + .../service/client/params/ActionTokensArgs.java | 78 + .../service/client/params/ActionUpdateArgs.java | 32 + .../yarn/service/client/params/ArgOps.java | 156 ++ .../yarn/service/client/params/Arguments.java | 103 ++ .../yarn/service/client/params/ClientArgs.java | 252 +++ .../yarn/service/client/params/CommonArgs.java | 282 +++ .../client/params/ComponentArgsDelegate.java| 52 + .../client/params/DontSplitArguments.java | 34 + .../client/params/LaunchArgsAccessor.java | 30 + .../client/params/LaunchArgsDelegate.java | 51 + .../client/params/OptionArgsDelegate.java | 66 + .../client/params/PathArgumentConverter.java| 34 +
[64/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java new file mode 100644 index 000..cb7131e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -0,0 +1,494 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.component; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.apache.hadoop.yarn.service.ContainerFailureTracker; +import org.apache.hadoop.yarn.service.ServiceContext; +import org.apache.hadoop.yarn.service.ServiceScheduler; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.ServiceMetrics; +import org.apache.hadoop.yarn.state.InvalidStateTransitionException; +import org.apache.hadoop.yarn.state.MultipleArcTransition; +import org.apache.hadoop.yarn.state.SingleArcTransition; +import org.apache.hadoop.yarn.state.StateMachine; +import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils; +import org.apache.hadoop.yarn.service.monitor.probe.Probe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*; +import static org.apache.hadoop.yarn.service.component.ComponentEventType.*; +import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START; +import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP; +import static org.apache.hadoop.yarn.service.component.ComponentState.*; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD; + +public class Component implements EventHandler { + private static final Logger LOG = LoggerFactory.getLogger(Component.class); + + private org.apache.hadoop.yarn.service.api.records.Component componentSpec; + private long allocateId; + private Priority priority; + private ServiceMetrics componentMetrics; + private ServiceScheduler scheduler; + private ServiceContext context; + private AMRMClientAsync amrmClient; + private AtomicLong instanceIdCounter = new AtomicLong(); + private MapcompInstances = + new ConcurrentHashMap<>(); + // component instances to be assigned with a container + private List pendingInstances = new LinkedList<>(); + private ContainerFailureTracker
[22/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java deleted file mode 100644 index d96d13e..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.hadoop.yarn.service.provider.ProviderService; -import org.apache.hadoop.yarn.service.provider.ProviderFactory; -import org.apache.slider.server.appmaster.actions.QueueAccess; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.services.workflow.ServiceThreadFactory; -import org.apache.slider.server.services.workflow.WorkflowExecutorService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY; - -/** - * A service for launching containers - */ -public class RoleLaunchService -extends WorkflowExecutorService { - protected static final Logger log = -LoggerFactory.getLogger(RoleLaunchService.class); - - public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService"; - - - /** - * Queue submission API - */ - private QueueAccess actionQueue; - - /** - * Filesystem to use for the launch - */ - private SliderFileSystem fs; - - - private MapenvVars = new HashMap<>(); - - /** - * Construct an instance of the launcher - * @param queueAccess - * @param fs filesystem - * @param envVars environment variables - */ - public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs, - Map envVars) { -super(ROLE_LAUNCH_SERVICE); -this.actionQueue = queueAccess; -this.fs = fs; -this.envVars = envVars; - } - - public RoleLaunchService(SliderFileSystem fs) { -super(ROLE_LAUNCH_SERVICE); -this.fs = fs; - } - - @Override - public void init(Configuration conf) { -super.init(conf); -setExecutor(Executors.newCachedThreadPool( -new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true))); - } - - /** - * Start an asychronous launch operation - * @param assignment container assignment - * @param credentials credentials to use - */ - public void launchRole(ContainerAssignment assignment, - Application application, Credentials credentials) { - } - - public void launchComponent(Application application, - ComponentInstance instance, Container container) { -RoleLaunchService.RoleLauncher launcher = -new RoleLaunchService.RoleLauncher(application, instance, -container); -execute(launcher); - } - - /** - * Thread that runs on the AM to launch a container - */ - private class RoleLauncher implements Runnable { -// Allocated container -public final Container container; -public final Application application; -
[18/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java deleted file mode 100644 index 2e40a9b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.rpc; - -import com.google.common.base.Preconditions; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.ipc.ProtobufHelper; -import org.apache.hadoop.ipc.ProtocolSignature; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.api.proto.Messages; - -import java.io.IOException; -import java.net.InetSocketAddress; - -public class SliderClusterProtocolProxy implements SliderClusterProtocol { - - private static final RpcController NULL_CONTROLLER = null; - private final SliderClusterProtocolPB endpoint; - private final InetSocketAddress address; - - public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint, - InetSocketAddress address) { -Preconditions.checkArgument(endpoint != null, "null endpoint"); -Preconditions.checkNotNull(address != null, "null address"); -this.endpoint = endpoint; -this.address = address; - } - - @Override - public String toString() { -final StringBuilder sb = -new StringBuilder("SliderClusterProtocolProxy{"); -sb.append("address=").append(address); -sb.append('}'); -return sb.toString(); - } - - @Override - public ProtocolSignature getProtocolSignature(String protocol, - long clientVersion, - int clientMethodsHash) - throws IOException { -if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) { - throw new IOException("Serverside implements " + -RPC.getProtocolName(SliderClusterProtocolPB.class) + -". The following requested protocol is unknown: " + -protocol); -} - -return ProtocolSignature.getProtocolSignature(clientMethodsHash, -RPC.getProtocolVersion( -SliderClusterProtocol.class), -SliderClusterProtocol.class); - } - - @Override - public long getProtocolVersion(String protocol, long clientVersion) - throws IOException { -return SliderClusterProtocol.versionID; - } - - private IOException convert(ServiceException se) { -IOException ioe = ProtobufHelper.getRemoteException(se); -if (ioe instanceof RemoteException) { - RemoteException remoteException = (RemoteException) ioe; - return remoteException.unwrapRemoteException(); -} -return ioe; - } - - @Override public Messages.StopClusterResponseProto stopCluster( - Messages.StopClusterRequestProto request) - throws IOException, YarnException { -try { - return endpoint.stopCluster(NULL_CONTROLLER, request); -} catch (ServiceException e) { - throw convert(e); -} - } - - @Override - public Messages.UpgradeContainersResponseProto upgradeContainers( - Messages.UpgradeContainersRequestProto request) throws IOException, - YarnException { -try { - return endpoint.upgradeContainers(NULL_CONTROLLER, request); -} catch (ServiceException e) { - throw convert(e); -} - } - -
[69/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index 88f74ef..17f8c95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -22,7 +22,7 @@ info: Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like. -To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration. +To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration. We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management. @@ -41,177 +41,177 @@ schemes: - http - https # will be prefixed to all paths -basePath: /services/v1/ +basePath: /ws/v1/ consumes: - application/json produces: - application/json paths: - /applications: + /services: get: - summary: List of applications/services running in the cluster - description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name. + summary: List of services running in the cluster + description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. responses: 200: - description: An array of applications + description: An array of services schema: type: array items: - $ref: '#/definitions/Application' + $ref: '#/definitions/Service' default: description: Unexpected error schema: -$ref: '#/definitions/ApplicationStatus' +$ref: '#/definitions/ServiceStatus' post: - summary: Create an application/service - description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state. + summary: Create a service + description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only
[57/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java new file mode 100644 index 000..b742553 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.timelineservice; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.ServiceContext; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.ContainerState; +import org.apache.hadoop.yarn.service.api.records.PlacementPolicy; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test class for ServiceTimelinePublisher. + */ +public class TestServiceTimelinePublisher { + private TimelineV2Client timelineClient; + private Configuration config; + private ServiceTimelinePublisher serviceTimelinePublisher; + private static String SERVICE_NAME = "HBASE"; + private static String SERVICEID = "application_1490093646524_0005"; + private static String ARTIFACTID = "ARTIFACTID"; + private static String COMPONENT_NAME = "DEFAULT"; + private static String CONTAINER_ID = + "container_e02_1490093646524_0005_01_01"; + private static String CONTAINER_IP = + "localhost"; + private static String CONTAINER_HOSTNAME = + "cnl124-localhost.site"; + private static String CONTAINER_BAREHOST = + "localhost.com"; + + @Before + public void setUp() throws Exception { +config = new Configuration(); +config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); +config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); +timelineClient = +new DummyTimelineClient(ApplicationId.fromString(SERVICEID)); +serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient); +timelineClient.init(config); +serviceTimelinePublisher.init(config); +timelineClient.start(); +serviceTimelinePublisher.start(); + } + + @After + public void tearDown() throws Exception { +if
[27/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java deleted file mode 100644 index fc57c82..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ /dev/null @@ -1,2548 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import com.google.common.base.Preconditions; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.GlobFilter; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.Slider; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; -import org.apache.hadoop.yarn.service.client.params.Arguments; -import org.apache.hadoop.yarn.service.client.params.SliderActions; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.ClasspathConstructor; -import org.apache.slider.core.main.LauncherExitCodes; -import org.apache.slider.server.services.utility.PatternValidator; -import org.apache.slider.server.services.workflow.ForkedProcessService; -import org.apache.zookeeper.server.util.KerberosUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.io.Serializable; -import java.io.StringWriter; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLDecoder; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.EnumSet; -import
[03/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java deleted file mode 100644 index 9c5708f..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.mock; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.junit.Assert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertNotNull; - -/** - * This is an evolving engine to mock YARN operations. - */ -public class MockYarnEngine { - private static final Logger LOG = - LoggerFactory.getLogger(MockYarnEngine.class); - - private MockYarnCluster cluster; - private Allocator allocator; - private List pending = new ArrayList<>(); - - private ApplicationId appId = new MockApplicationId(0, 0); - - private ApplicationAttemptId attemptId = new MockApplicationAttemptId(appId, - 1); - - @Override - public String toString() { -return "MockYarnEngine " + cluster + " + pending=" + pending.size(); - } - - public int containerCount() { -return cluster.containersInUse(); - } - - public MockYarnEngine(int clusterSize, int containersPerNode) { -cluster = new MockYarnCluster(clusterSize, containersPerNode); -allocator = new Allocator(cluster); - } - - public MockYarnCluster getCluster() { -return cluster; - } - - public Allocator getAllocator() { -return allocator; - } - - /** - * Allocate a container from a request. The containerID will be - * unique, nodeId and other fields chosen internally with - * no such guarantees; resource and priority copied over - * @param request request - * @return container - */ - public Container allocateContainer(AMRMClient.ContainerRequest request) { -MockContainer allocated = allocator.allocate(request); -if (allocated != null) { - MockContainerId id = (MockContainerId)allocated.getId(); - id.setApplicationAttemptId(attemptId); -} -return allocated; - } - - MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId - containerId) { -return cluster.release(containerId); - } - - /** - * Process a list of operations -release containers to be released, - * allocate those for which there is space (but don't rescan the list after - * the scan). - * @param ops - * @return - */ - public List execute(List ops) { -return execute(ops, new ArrayList<>()); - } - - /** - * Process a list of operations -release containers to be released, - * allocate those for which there is space (but don't rescan the list after - * the scan). Unsatisifed entries
[39/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java new file mode 100644 index 000..e4eae20 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.containerlaunch; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerRetryContext; +import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.utils.CoreFileSystem; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK; + +/** + * Launcher of applications: base class + */ +public class AbstractLauncher { + private static final Logger log = +LoggerFactory.getLogger(AbstractLauncher.class); + public static final String CLASSPATH = "CLASSPATH"; + /** + * Filesystem to use for the launch + */ + protected final CoreFileSystem coreFileSystem; + /** + * Env vars; set up at final launch stage + */ + protected final MapenvVars = new HashMap<>(); + protected final ContainerLaunchContext containerLaunchContext = +Records.newRecord(ContainerLaunchContext.class); + protected final List commands = new ArrayList<>(20); + protected final Map localResources = new HashMap<>(); + protected final Map mountPaths = new HashMap<>(); + private final Map serviceData = new HashMap<>(); + // security + protected final Credentials credentials; + protected boolean yarnDockerMode = false; + protected String dockerImage; + protected String dockerNetwork = DEFAULT_DOCKER_NETWORK; + protected String dockerHostname; + protected String runPrivilegedContainer; + + + /** + * Create instance. + * @param coreFileSystem filesystem + * @param credentials initial set of credentials -null is permitted + */ + public AbstractLauncher( + CoreFileSystem coreFileSystem, + Credentials credentials) { +this.coreFileSystem = coreFileSystem; +this.credentials = credentials != null ? credentials: new Credentials(); + } + + public void setYarnDockerMode(boolean yarnDockerMode){ +this.yarnDockerMode = yarnDockerMode; + } + + /** + * Get the env vars to work on + * @return env vars + */ + public Map getEnv() { +return envVars; + } + + /** + * Get the launch commands. + * @return the live list of commands + */ + public List getCommands() { +return commands; + } + + public void addLocalResource(String subPath, LocalResource resource) { +localResources.put(subPath, resource); + } + + public void addLocalResource(String subPath, LocalResource resource, String mountPath) { +localResources.put(subPath,
[24/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java deleted file mode 100644 index 1964459..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.persist; - -public class PersistKeys { - - public static final String SCHEMA = -"http://example.org/specification/v2.0.0;; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java deleted file mode 100644 index ac8fca5..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.registry; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.yarn.service.conf.SliderKeys; - -/** - * Miscellaneous methods to assist slider registry work - * - */ -public class SliderRegistryUtils { - - - /** - * Get the registry path for an instance under the user's home node - * @param instanceName application instance - * @return a path to the registry location for this application instance. - */ - public static String registryPathForInstance(String instanceName) { -return RegistryUtils.servicePath( -RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName -); - } - - /** - * Process a path expanding it if needed. - * Validation is delegated to later as the core registry will need - * to do that anyway - * @param path path - * @return a path maybe with some expansion - */ - public static String resolvePath(String path) { -Preconditions.checkArgument(path!=null, "null path"); -Preconditions.checkArgument(!path.isEmpty(), "empty path"); -String newpath = path; -if (path.startsWith("~/")) { - // add user expansion - newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1); -} else if (path.equals("~")) { - newpath =
[37/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java new file mode 100644 index 000..b851fb7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.servicemonitor.probe; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; + +import java.io.IOException; +import java.util.Map; + +/** + * Base class of all probes. + */ +public abstract class Probe implements MonitorKeys { + + protected final Configuration conf; + private String name; + + /** + * Create a probe of a specific name + * + * @param name probe name + * @param conf configuration being stored. + */ + public Probe(String name, Configuration conf) { +this.name = name; +this.conf = conf; + } + + + protected void setName(String name) { +this.name = name; + } + + public String getName() { +return name; + } + + + @Override + public String toString() { +return getName(); + } + + public static String getProperty(Mapprops, String name, + String defaultValue) throws IOException { +String value = props.get(name); +if (StringUtils.isEmpty(value)) { + if (defaultValue == null) { +throw new IOException(name + " not specified"); + } + return defaultValue; +} +return value; + } + + public static int getPropertyInt(Map props, String name, + Integer defaultValue) throws IOException { +String value = props.get(name); +if (StringUtils.isEmpty(value)) { + if (defaultValue == null) { +throw new IOException(name + " not specified"); + } + return defaultValue; +} +return Integer.parseInt(value); + } + + /** + * perform any prelaunch initialization + */ + public void init() throws IOException { + + } + + /** + * Ping the endpoint. All exceptions must be caught and included in the + * (failure) status. + * + * @param instance instance to ping + * @return the status + */ + public abstract ProbeStatus ping(ComponentInstance instance); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java new file mode 100644 index 000..7cd761c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache
[63/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java new file mode 100644 index 000..7baa284 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.containerlaunch; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Build a single command line to include in the container commands; + * Special support for JVM command buildup. + */ +public class CommandLineBuilder { + protected final List argumentList = new ArrayList<>(20); + + /** + * Add an entry to the command list + * @param args arguments -these will be converted strings + */ + public void add(Object... args) { +for (Object arg : args) { + argumentList.add(arg.toString()); +} + } + + // Get the number of arguments + public int size() { +return argumentList.size(); + } + + /** + * Append the output and error files to the tail of the command + * @param stdout out + * @param stderr error. Set this to null to append into stdout + */ + public void addOutAndErrFiles(String stdout, String stderr) { +Preconditions.checkNotNull(stdout, "Null output file"); +Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); +// write out the path output +argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stdout); +if (stderr != null) { + argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stderr); +} else { + argumentList.add("2>&1"); +} + } + + /** + * This just returns the command line + * @see #build() + * @return the command line + */ + @Override + public String toString() { +return build(); + } + + /** + * Build the command line + * @return the command line + */ + public String build() { +return SliderUtils.join(argumentList, " "); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java new file mode 100644 index 000..0e51a62 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0
[16/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java deleted file mode 100644 index 3e8a3c3..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.state; - -import org.apache.hadoop.yarn.api.records.Container; - -/** - * Static assignment structure - */ -public class ContainerAssignment { - - /** - * Container that has been allocated - */ - public final Container container; - - /** - * Role to assign to it - */ - public final RoleStatus role; - - /** - * Placement outcome: was this from history or not - */ - public final ContainerAllocationOutcome placement; - - public ContainerAssignment(Container container, - RoleStatus role, - ContainerAllocationOutcome placement) { -this.container = container; -this.role = role; -this.placement = placement; - } - - @Override - public String toString() { -final StringBuilder sb = new StringBuilder("ContainerAssignment{"); -sb.append("container=").append(container); -sb.append(", role=").append(role); -sb.append(", placement=").append(placement); -sb.append('}'); -return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java deleted file mode 100644 index 6df4bf4..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.state; - -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; - -/** - * Container outcomes we care about; slightly simplified from - * {@link ContainerExitStatus} -and hopefully able to handle - * any new exit codes. - */ -public enum ContainerOutcome { - Completed, - Failed, - Failed_limits_exceeded, - Disk_failure, - Preempted; - - /** - * Build a container outcome from
[47/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java deleted file mode 100644 index 405f690..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.io.IOException; - -/** - * Serialized form of an application report which can be persisted - * and then parsed. It can not be converted back into a - * real YARN application report - * - * Useful for testing - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) - -public class SerializedApplicationReport { - - public String applicationId; - public String applicationAttemptId; - public String name; - public String applicationType; - public String user; - public String queue; - public String host; - public Integer rpcPort; - public String state; - public String diagnostics; - public String url; - /** - * This value is non-null only when a report is generated from a submission context. - * The YARN {@link ApplicationReport} structure does not propagate this value - * from the RM. - */ - public Long submitTime; - public Long startTime; - public Long finishTime; - public String finalStatus; - public String origTrackingUrl; - public Float progress; - - public SerializedApplicationReport() { - } - - public SerializedApplicationReport(ApplicationReport report) { -this.applicationId = report.getApplicationId().toString(); -ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); -this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; -this.name = report.getName(); -this.applicationType = report.getApplicationType(); -this.user = report.getUser(); -this.queue = report.getQueue(); -this.host = report.getHost(); -this.rpcPort = report.getRpcPort(); -this.state = report.getYarnApplicationState().toString(); -this.diagnostics = report.getDiagnostics(); -this.startTime = report.getStartTime(); -this.finishTime = report.getFinishTime(); -FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); -this.finalStatus = appStatus == null ? "" : appStatus.toString(); -this.progress = report.getProgress(); -this.url = report.getTrackingUrl(); -this.origTrackingUrl= report.getOriginalTrackingUrl(); - } - - @Override - public String toString() { -try { - return ApplicationReportSerDeser.toString(this); -} catch (IOException e) { - return super.toString(); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java -- diff --git
[04/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java deleted file mode 100644 index 9632265..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java +++ /dev/null @@ -1,539 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.mock; - -import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderInternalStateException; -import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.core.main.LauncherExitCodes; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.ContainerOutcome; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.NodeMap; -import org.apache.slider.server.appmaster.state.ProviderAppState; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.slider.utils.SliderTestBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map.Entry; - -/** - * Base for app state tests. - */ -public abstract class BaseMockAppStateTest extends SliderTestBase implements -MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(BaseMockAppStateTest.class); - protected static final List EMPTY_ID_LIST = Collections - .emptyList(); - - protected final MockFactory factory = MockFactory.INSTANCE; - protected MockAppState appState; - protected MockYarnEngine engine; - protected FileSystem fs; - protected SliderFileSystem sliderFileSystem; - protected File historyWorkDir; - protected Path historyPath; - protected MockApplicationId applicationId; - protected
[58/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java new file mode 100644 index 000..be36335 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java @@ -0,0 +1,530 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.utils.JsonSerDeser; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME; +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; +import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*; +import static org.easymock.EasyMock.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test for ServiceApiUtil helper methods. + */ +public class TestServiceApiUtil { + private static final Logger LOG = LoggerFactory + .getLogger(TestServiceApiUtil.class); + private static final String EXCEPTION_PREFIX = "Should have thrown " + + "exception: "; + private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + + "exception: "; + + private static final String LEN_64_STR = + "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01"; + + private static final YarnConfiguration CONF_DEFAULT_DNS = new + YarnConfiguration(); + private static final YarnConfiguration CONF_DNS_ENABLED = new + YarnConfiguration(); + + @BeforeClass + public static void init() { +CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true); + } + + @Test(timeout = 9) + public void testResourceValidation() throws Exception { +assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR +.length()); + +SliderFileSystem sfs = initMock(null); + +Service app = new Service(); + +// no name +try { + ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); + Assert.fail(EXCEPTION_PREFIX + "service with no name"); +} catch (IllegalArgumentException e) { + assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); +} + +// bad format name +String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR}; +for (String badName : badNames) { + app.setName(badName); + try { +ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); +Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName); + } catch (IllegalArgumentException e) { +
[62/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java new file mode 100644 index 000..684f655 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.monitor.probe; + +import org.apache.hadoop.yarn.service.api.records.ReadinessCheck; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Formatter; +import java.util.Locale; + +/** + * Various utils to work with the monitor + */ +public final class MonitorUtils { + protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils + .class); + + private MonitorUtils() { + } + + public static String toPlural(int val) { +return val != 1 ? "s" : ""; + } + + /** + * Convert milliseconds to human time -the exact format is unspecified + * @param milliseconds a time in milliseconds + * @return a time that is converted to human intervals + */ + public static String millisToHumanTime(long milliseconds) { +StringBuilder sb = new StringBuilder(); +// Send all output to the Appendable object sb +Formatter formatter = new Formatter(sb, Locale.US); + +long s = Math.abs(milliseconds / 1000); +long m = Math.abs(milliseconds % 1000); +if (milliseconds > 0) { + formatter.format("%d.%03ds", s, m); +} else if (milliseconds == 0) { + formatter.format("0"); +} else { + formatter.format("-%d.%03ds", s, m); +} +return sb.toString(); + } + + public static Probe getProbe(ReadinessCheck readinessCheck) { +if (readinessCheck == null) { + return null; +} +if (readinessCheck.getType() == null) { + return null; +} +try { + switch (readinessCheck.getType()) { + case HTTP: +return HttpProbe.create(readinessCheck.getProps()); + case PORT: +return PortProbe.create(readinessCheck.getProps()); + default: +return null; + } +} catch (Throwable t) { + throw new IllegalArgumentException("Error creating readiness check " + + t); +} + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java new file mode 100644 index 000..aba5859 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You
[10/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json deleted file mode 100644 index da06c13..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "roleCommand": "{{COMMAND}}", - "clusterName": "{{CLUSTER_NAME}}", - "hostname": "{{HOST_NAME}}", - "hostLevelParams": { -"java_home": "/usr/jdk64/jdk1.7.0_45" - }, - "commandType": "EXECUTION_COMMAND", - "roleParams": {}, - "serviceName": "{{SERVICE_NAME}}", - "role": "{{ROLE_NAME}}", - "commandParams": {}, - "taskId": "{{TASK_ID}}", - "public_hostname": "{{HOST_NAME}}", - "configurations": { -"hbase-log4j": { - "log4j.threshold": "ALL", - "log4j.rootLogger": "${hbase.root.logger}", - "log4j.logger.org.apache.zookeeper": "INFO", - "log4j.logger.org.apache.hadoop.hbase": "DEBUG", - "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO", - "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO", - "log4j.category.SecurityLogger": "${hbase.security.logger}", - "log4j.appender.console": "org.apache.log4j.ConsoleAppender", - "log4j.appender.console.target": "System.err", - "log4j.appender.console.layout": "org.apache.log4j.PatternLayout", - "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender", - "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout", - "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n", - "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}", - "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}", - "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}", - "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender", - "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout", - "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}", - "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}", - "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}", - "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender", - "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender", - "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout", - "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}", - "log4j.appender.DRFA.DatePattern": ".-MM-dd", - "log4j.additivity.SecurityLogger": "false", - "hbase.security.logger": "INFO,console", - "hbase.security.log.maxfilesize": "256MB", - "hbase.security.log.maxbackupindex": "20", - "hbase.security.log.file": "SecurityAuth.audit", - "hbase.root.logger": "INFO,console", - "hbase.log.maxfilesize": "256MB", - "hbase.log.maxbackupindex": "20", - "hbase.log.file": "hbase.log", - "hbase.log.dir": "." -}, -"global": { - "hbase_root": "{{HBASE_HOME}}", - "hbase_pid_dir": "{{PID_DIR}}", - "proxyuser_group": "users", - "syncLimit": "5", - "hbase_regionserver_heapsize": "{{REGION_SERVER_HEAP_SIZE}}", - "rca_enabled": "false", - "tickTime": "2000", - "hbase_master_heapsize": "{{MASTER_HEAP_SIZE}}", - "initLimit": "10", - "user_group": "{{GROUP_NAME}}", - "hbase_user": "{{USER_NAME}}", - "hbase_log_dir": "{{LOG_DIR}}" -}, -"hdfs-site": { - "dfs.namenode.checkpoint.period": "21600", - "dfs.namenode.avoid.write.stale.datanode": "true", - "dfs.namenode.checkpoint.txns": "100", - "dfs.block.access.token.enable": "true", - "dfs.support.append": "true", - "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}", - "dfs.cluster.administrators": " hdfs", - "dfs.replication": "3", - "ambari.dfs.datanode.http.port": "50075", - "dfs.datanode.balance.bandwidthPerSec": "625", - "dfs.namenode.safemode.threshold-pct": "1.0f", -
[52/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java deleted file mode 100644 index 982a114..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java +++ /dev/null @@ -1,493 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.compinstance; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.client.api.NMClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.service.ServiceScheduler; -import org.apache.hadoop.yarn.service.api.records.ContainerState; -import org.apache.hadoop.yarn.service.component.Component; -import org.apache.hadoop.yarn.state.InvalidStateTransitionException; -import org.apache.hadoop.yarn.state.SingleArcTransition; -import org.apache.hadoop.yarn.state.StateMachine; -import org.apache.hadoop.yarn.state.StateMachineFactory; -import org.apache.hadoop.yarn.util.BoundedAppender; -import org.apache.hadoop.yarn.service.utils.SliderUtils; -import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; -import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Date; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; - -import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER; -import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*; - -public class ComponentInstance implements EventHandler, -Comparable { - private static final Logger LOG = - LoggerFactory.getLogger(ComponentInstance.class); - - private StateMachinestateMachine; - private Component component; - private final ReadLock readLock; - private final WriteLock writeLock; - - private ComponentInstanceId compInstanceId = null; - private Path compInstanceDir; - private Container container; - private YarnRegistryViewForProviders yarnRegistryOperations; - private FileSystem fs; - private boolean timelineServiceEnabled = false; - private
[01/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He [Forced Update!]
Repository: hadoop Updated Branches: refs/heads/yarn-native-services 6e7ddba0c -> d924a4075 (forced update) http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java deleted file mode 100644 index 5493198..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.utils; - -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.compress.utils.IOUtils; -import org.junit.Assert; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; - -/** - * Various utility methods - * Byte comparison methods are from - * org.apache.hadoop.fs.contract.ContractTestUtils - */ -public class TestUtility { - protected static final Logger log = - LoggerFactory.getLogger(TestUtility.class); - - public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException { -for (File file : dirObj.listFiles()) { - if (file.isDirectory()) { -addDir(file, zipFile, prefix + file.getName() + File.separator); - } else { -log.info("Adding to zip - " + prefix + file.getName()); -zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName())); -IOUtils.copy(new FileInputStream(file), zipFile); -zipFile.closeArchiveEntry(); - } -} - } - - public static void zipDir(String zipFile, String dir) throws IOException { -File dirObj = new File(dir); -ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile)); -log.info("Creating : {}", zipFile); -try { - addDir(dirObj, out, ""); -} finally { - out.close(); -} - } - - public static String createAppPackage( - TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException { -String zipFileName; -File pkgPath = folder.newFolder(subDir); -File zipFile = new File(pkgPath, pkgName).getAbsoluteFile(); -zipFileName = zipFile.getAbsolutePath(); -TestUtility.zipDir(zipFileName, srcPath); -log.info("Created temporary zip file at {}", zipFileName); -return zipFileName; - } - - - /** - * Assert that tthe array original[0..len] and received[] are equal. - * A failure triggers the logging of the bytes near where the first - * difference surfaces. - * @param original source data - * @param received actual - * @param len length of bytes to compare - */ - public static void compareByteArrays(byte[] original, - byte[] received, - int len) { -Assert.assertEquals("Number of bytes read != number written", -len, received.length); -int errors = 0; -int first_error_byte = -1; -for (int i = 0; i < len; i++) { - if (original[i] != received[i]) { -if (errors == 0) { - first_error_byte = i; -} -errors++; - } -} - -if (errors > 0) { - String message = String.format(" %d errors in file of length %d", - errors, len); - log.warn(message); - // the range either side of the first error to print - // this is a purely arbitrary number, to aid user debugging - final int overlap = 10; - for (int i = Math.max(0, first_error_byte - overlap); - i <
[51/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java deleted file mode 100644 index 7baa284..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.containerlaunch; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.service.utils.SliderUtils; - -import java.util.ArrayList; -import java.util.List; - -/** - * Build a single command line to include in the container commands; - * Special support for JVM command buildup. - */ -public class CommandLineBuilder { - protected final List argumentList = new ArrayList<>(20); - - /** - * Add an entry to the command list - * @param args arguments -these will be converted strings - */ - public void add(Object... args) { -for (Object arg : args) { - argumentList.add(arg.toString()); -} - } - - // Get the number of arguments - public int size() { -return argumentList.size(); - } - - /** - * Append the output and error files to the tail of the command - * @param stdout out - * @param stderr error. Set this to null to append into stdout - */ - public void addOutAndErrFiles(String stdout, String stderr) { -Preconditions.checkNotNull(stdout, "Null output file"); -Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); -// write out the path output -argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stdout); -if (stderr != null) { - argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stderr); -} else { - argumentList.add("2>&1"); -} - } - - /** - * This just returns the command line - * @see #build() - * @return the command line - */ - @Override - public String toString() { -return build(); - } - - /** - * Build the command line - * @return the command line - */ - public String build() { -return SliderUtils.join(argumentList, " "); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java deleted file mode 100644 index fcbb69b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you
[45/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java deleted file mode 100644 index 8310530..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.conf; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collection; - -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; -import static org.easymock.EasyMock.*; - -/** - * Test loading example resources. - */ -@RunWith(value = Parameterized.class) -public class TestLoadExampleAppJson extends Assert { - private String resource; - - public TestLoadExampleAppJson(String resource) { -this.resource = resource; - } - - @Parameterized.Parameters - public static Collectionfilenames() { -String[][] stringArray = new String[ExampleAppJson -.ALL_EXAMPLE_RESOURCES.size()][1]; -int i = 0; -for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) { - stringArray[i++][0] = s; -} -return Arrays.asList(stringArray); - } - - @Test - public void testLoadResource() throws Throwable { -try { - Application application = JSON_SER_DESER.fromResource(resource); - - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - replay(sfs, mockFs); - - ServiceApiUtil.validateAndResolveApplication(application, sfs, - new YarnConfiguration()); -} catch (Exception e) { - throw new Exception("exception loading " + resource + ":" + e.toString()); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java deleted file mode 100644 index 98c78d3..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you
[25/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java deleted file mode 100644 index 1fd49ab..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import org.apache.hadoop.yarn.client.ClientRMProxy; -import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.conf.HAUtil; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.text.DateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; - -/** - * Utils to work with credentials and tokens. - * - * Designed to be movable to Hadoop core - */ -public final class CredentialUtils { - - public static final String JOB_CREDENTIALS_BINARY - = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY; - - private CredentialUtils() { - } - - private static final Logger LOG = - LoggerFactory.getLogger(CredentialUtils.class); - - /** - * Save credentials to a byte buffer. Returns null if there were no - * credentials to save - * @param credentials credential set - * @return a byte buffer of serialized tokens - * @throws IOException if the credentials could not be written to the stream - */ - public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException { -ByteBuffer buffer = null; -if (!credentials.getAllTokens().isEmpty()) { - DataOutputBuffer dob = new DataOutputBuffer(); - try { -credentials.writeTokenStorageToStream(dob); - } finally { -dob.close(); - } - buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); -} -return buffer; - } - - public static File locateEnvCredentials(Mapenv, - Configuration conf, - StringBuffer sourceTextOut) throws FileNotFoundException { -String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION); -String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION; -if (tokenFilename == null) { - tokenFilename =
[20/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java deleted file mode 100644 index 7446e82..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.actions; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.server.appmaster.SliderAppMaster; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.RMOperationHandler; -import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions; -import org.apache.slider.server.appmaster.state.AppState; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * Kill a specific container - */ -public class ActionKillContainer extends AsyncAction { - - /** - * container to kill - */ - private final ContainerId containerId; - - /** - * handler for the operation - */ - private final RMOperationHandlerActions operationHandler; - - /** - * Kill a container - * @param containerId container to kill - * @param delay - * @param timeUnit - * @param operationHandler - */ - public ActionKillContainer( - ContainerId containerId, - long delay, - TimeUnit timeUnit, - RMOperationHandlerActions operationHandler) { -super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE); -this.operationHandler = operationHandler; -Preconditions.checkArgument(containerId != null); - -this.containerId = containerId; - } - - /** - * Get the container ID to kill - * @return - */ - public ContainerId getContainerId() { -return containerId; - } - - @Override - public void execute(SliderAppMaster appMaster, - QueueAccess queueService, - AppState appState) throws Exception { - List opsList = new LinkedList<>(); -ContainerReleaseOperation release = new ContainerReleaseOperation(containerId); -opsList.add(release); -//now apply the operations -operationHandler.execute(opsList); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java deleted file mode 100644 index 0d7f7d4..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional
[19/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java deleted file mode 100644 index 864a1cf..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.management; - -import com.codahale.metrics.JmxReporter; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.MetricSet; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.Slf4jReporter; -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.CompositeService; -import org.apache.slider.server.services.workflow.ClosingService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * YARN service which hooks up Codahale metrics to - * JMX, and, if enabled Ganglia and/or an SLF4J log. - */ -public class MetricsBindingService extends CompositeService -implements MetricsKeys { - protected static final Logger log = - LoggerFactory.getLogger(MetricsBindingService.class); - private final MetricRegistry metrics; - - private String reportingDetails = "not started"; - - - public MetricsBindingService(String name, - MetricRegistry metrics) { -super(name); -Preconditions.checkArgument(metrics != null, "Null metrics"); -this.metrics = metrics; - } - - /** - * Instantiate...create a metric registry in the process - * @param name service name - */ - public MetricsBindingService(String name) { -this(name, new MetricRegistry()); - } - - /** - * Accessor for the metrics instance - * @return the metrics - */ - public MetricRegistry getMetrics() { -return metrics; - } - - @Override - protected void serviceStart() throws Exception { -super.serviceStart(); - -StringBuilder summary = new StringBuilder(); -Configuration conf = getConfig(); - -summary.append("Reporting to JMX"); -// always start the JMX binding -JmxReporter jmxReporter; -jmxReporter = JmxReporter.forRegistry(metrics).build(); -jmxReporter.start(); -addService(new ClosingService<>(jmxReporter)); - - -// Ganglia -if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) { - log.warn("Ganglia integration is not implemented"); -/* - // This is all disabled due to transitive dependencies on an LGPL library - com.codahale.metrics.ganglia.GangliaReporter gangliaReporter; - String host = conf.getTrimmed(METRICS_GANGLIA_HOST, ""); - int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT); - int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60); - int ttl = 1; - info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode - mcast = info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host); - boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true); - - final info.ganglia.gmetric4j.gmetric.GMetric ganglia = - new info.ganglia.gmetric4j.gmetric.GMetric( - host, - port, - mcast, - ttl, - ganglia31); - gangliaReporter = com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics) -
[67/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java new file mode 100644 index 000..c64b1b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import io.swagger.annotations.ApiModelProperty; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +public class Error { + + private Integer code = null; + private String message = null; + private String fields = null; + + /** + **/ + public Error code(Integer code) { +this.code = code; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("code") + public Integer getCode() { +return code; + } + + public void setCode(Integer code) { +this.code = code; + } + + /** + **/ + public Error message(String message) { +this.message = message; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("message") + public String getMessage() { +return message; + } + + public void setMessage(String message) { +this.message = message; + } + + /** + **/ + public Error fields(String fields) { +this.fields = fields; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("fields") + public String getFields() { +return fields; + } + + public void setFields(String fields) { +this.fields = fields; + } + + @Override + public boolean equals(java.lang.Object o) { +if (this == o) { + return true; +} +if (o == null || getClass() != o.getClass()) { + return false; +} +Error error = (Error) o; +return Objects.equals(this.code, error.code) +&& Objects.equals(this.message, error.message) +&& Objects.equals(this.fields, error.fields); + } + + @Override + public int hashCode() { +return Objects.hash(code, message, fields); + } + + @Override + public String toString() { +StringBuilder sb = new StringBuilder(); +sb.append("class Error {\n"); + +sb.append("code: ").append(toIndentedString(code)).append("\n"); +sb.append("message: ").append(toIndentedString(message)).append("\n"); +sb.append("fields: ").append(toIndentedString(fields)).append("\n"); +sb.append("}"); +return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { +if (o == null) { + return "null"; +} +return o.toString().replace("\n", "\n"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java -- diff --git
[40/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java new file mode 100644 index 000..e310f45 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = { SliderActions.ACTION_UPDATE}, +commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE) + +public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs { + + @Override + public String getActionName() { +return SliderActions.ACTION_UPDATE; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java index f7b7349..00151f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java @@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.service.client.params; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.ErrorStrings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java index d8d8ab4..204149b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java @@ -26,59 +26,38 @@ package org.apache.hadoop.yarn.service.client.params; * so it is easier to see what arguments are there */ public interface Arguments { - String ARG_ADDON = "--addon"; - String
[29/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java deleted file mode 100644 index 51bde7b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import java.util.HashSet; -import java.util.Set; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; -import org.apache.hadoop.yarn.service.client.params.SliderActions; - -@Parameters(commandNames = { SliderActions.ACTION_LIST}, -commandDescription = SliderActions.DESCRIBE_ACTION_LIST) - -public class ActionListArgs extends AbstractActionArgs { - @Override - public String getActionName() { -return SliderActions.ACTION_LIST; - } - - @Parameter(names = {ARG_LIVE}, - description = "List only live application instances") - public boolean live; - - @Parameter(names = {ARG_STATE}, - description = "list only applications in the specific YARN state") - public String state = ""; - - @Parameter(names = {ARG_VERBOSE}, - description = "print out information in details") - public boolean verbose = false; - - @Parameter(names = {ARG_CONTAINERS}, - description = "List containers of an application instance") - public boolean containers; - - @Parameter(names = {ARG_VERSION}, - description = "Filter containers by app version (used with " + -ARG_CONTAINERS + ")") - public String version; - - @Parameter(names = {ARG_COMPONENTS}, variableArity = true, - description = "Filter containers by component names (used with " + -ARG_CONTAINERS + ")") - public Set components = new HashSet<>(0); - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { -return 0; - } - - @Override - public int getMaxParams() { -return 1; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java deleted file mode 100644 index 012..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed
[13/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java deleted file mode 100644 index 3e9b764..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.publisher; - -import org.apache.hadoop.yarn.webapp.NotFoundException; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.PublishedConfigSet; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.registry.docstore.PublishedExports; -import org.apache.slider.core.registry.docstore.PublishedExportsSet; -import org.apache.slider.core.registry.docstore.UriMap; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.UriInfo; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_REGEXP; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP; - -/** - * This publishes configuration sets - */ -public class PublisherResource extends AbstractSliderResource { - protected static final Logger log = - LoggerFactory.getLogger(PublisherResource.class); - public static final String EXPORTS_NAME = "exports"; - public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME; - public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ; - public static final String SET_NAME = - "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}"; - public static final String SETNAME = "setname"; - public static final String CLASSPATH = "/classpath"; - public static final String CONFIG = "config"; - - public static final String SETNAME_PATTERN = - "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}"; - private static final String CONFIG_PATTERN = - SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + "}"; - private final StateAccessForProviders appState; - - public PublisherResource(WebAppApi slider) { -super(slider); -appState = slider.getAppState(); - } - - private void init(HttpServletResponse res, UriInfo uriInfo) { -res.setContentType(null); -log.debug(uriInfo.getRequestUri().toString()); - } - - /** - * Get a named config set - * @param setname name of the config set - * @return the config set - * @throws NotFoundException if there was no matching set - */ - private PublishedConfigSet getConfigSet(String setname) { -PublishedConfigSet configSet = -appState.getPublishedConfigSet(setname); -if (configSet == null) { -
[33/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java deleted file mode 100644 index e89306c..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.api.resource; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import org.apache.commons.lang.StringUtils; -import org.apache.slider.common.tools.SliderUtils; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Set of configuration properties that can be injected into the application - * components via envs, files and custom pluggable helper docker containers. - * Files of several standard formats like xml, properties, json, yaml and - * templates will be supported. - **/ - -@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.") -@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") -@JsonInclude(JsonInclude.Include.NON_NULL) -public class Configuration implements Serializable { - private static final long serialVersionUID = -4330788704981074466L; - - private Mapproperties = new HashMap (); - private Map env = new HashMap (); - private List files = new ArrayList(); - - /** - * A blob of key-value pairs of common application properties. - **/ - public Configuration properties(Map properties) { -this.properties = properties; -return this; - } - - @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.") - @JsonProperty("properties") - public Map getProperties() { -return properties; - } - - public void setProperties(Map properties) { -this.properties = properties; - } - - /** - * A blob of key-value pairs which will be appended to the default system - * properties and handed off to the application at start time. All placeholder - * references to properties will be substituted before injection. - **/ - public Configuration env(Map env) { -this.env = env; -return this; - } - - @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.") - @JsonProperty("env") - public Map getEnv() { -return env; - } - - public void setEnv(Map env) { -this.env = env; - } - - /** - * Array of list of files that needs to be created and made available as - * volumes in the application component containers. - **/ - public Configuration files(List files) { -this.files = files; -return this; - } - - @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.") -
[50/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java deleted file mode 100644 index 83c9961..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.provider; - -import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory; -import org.apache.hadoop.yarn.service.api.records.Artifact; -import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory; -import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for factories. - */ -public abstract class ProviderFactory { - protected static final Logger LOG = - LoggerFactory.getLogger(ProviderFactory.class); - - protected ProviderFactory() {} - - public abstract AbstractClientProvider createClientProvider(); - - public abstract ProviderService createServerProvider(); - - public static synchronized ProviderService getProviderService(Artifact - artifact) { -return createSliderProviderFactory(artifact).createServerProvider(); - } - - public static synchronized AbstractClientProvider getClientProvider(Artifact - artifact) { -return createSliderProviderFactory(artifact).createClientProvider(); - } - - /** - * Create a provider for a specific application - * @param artifact artifact - * @return provider factory - */ - public static synchronized ProviderFactory createSliderProviderFactory( - Artifact artifact) { -if (artifact == null || artifact.getType() == null) { - LOG.debug("Loading service provider type default"); - return DefaultProviderFactory.getInstance(); -} -LOG.debug("Loading service provider type {}", artifact.getType()); -switch (artifact.getType()) { - // TODO add handling for custom types? - // TODO handle application - case DOCKER: -return DockerProviderFactory.getInstance(); - case TARBALL: -return TarballProviderFactory.getInstance(); - default: -throw new IllegalArgumentException(String.format("Resolution error, " + -"%s should not be passed to createSliderProviderFactory", -artifact.getType())); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java deleted file mode 100644 index 9ef0176..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership.
[73/74] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi
YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0344e8ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0344e8ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0344e8ae Branch: refs/heads/yarn-native-services Commit: 0344e8ae7e64f6d6298c16b5ddf8476cbc4473a8 Parents: e94f23e Author: Jian HeAuthored: Tue Aug 29 11:09:00 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:22:31 2017 -0700 -- NOTICE.txt | 15 + .../resources/assemblies/hadoop-yarn-dist.xml |8 - .../assemblies/hadoop-yarn-services-api.xml | 36 - .../assemblies/hadoop-yarn-services-dist.xml| 30 - hadoop-project/pom.xml | 19 +- hadoop-yarn-project/hadoop-yarn/bin/yarn| 37 +- .../hadoop-yarn-services-api/pom.xml| 104 +- .../yarn/service/webapp/ApiServerWebApp.java|4 +- .../src/main/resources/webapps/api-server/app | 16 + .../resources/webapps/services-rest-api/app | 16 - .../hadoop-yarn-services-core/pom.xml | 213 +--- .../service/client/params/ActionKDiagArgs.java | 76 -- .../yarn/service/client/params/ClientArgs.java |5 - .../registry/YarnRegistryViewForProviders.java |8 +- .../yarn/service/utils/KerberosDiags.java | 680 --- .../hadoop/yarn/service/utils/SliderUtils.java | 1088 -- .../hadoop/yarn/service/ServiceTestUtils.java | 28 + .../hadoop/yarn/service/TestServiceApiUtil.java | 38 +- .../yarn/service/TestYarnNativeServices.java| 10 +- .../yarn/service/client/TestServiceCLI.java |1 - .../yarn/service/conf/TestAppJsonResolve.java | 30 +- .../service/conf/TestLoadExampleAppJson.java| 11 +- .../providers/TestAbstractClientProvider.java | 10 +- hadoop-yarn-project/pom.xml |4 + 24 files changed, 176 insertions(+), 2311 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344e8ae/NOTICE.txt -- diff --git a/NOTICE.txt b/NOTICE.txt index c41972b..e40f3bf 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -576,3 +576,18 @@ The binary distribution of this product bundles binaries of software.amazon.ion:ion-java 1.0.1, which has the following notices: * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +The binary distribution of this product bundles binaries of +JCommander (https://github.com/cbeust/jcommander), +which has the following notices: + * Copyright 2010 Cedric Beust ced...@beust.com + +The binary distribution of this product bundles binaries of +snakeyaml (https://bitbucket.org/asomov/snakeyaml), +which has the following notices: + * Copyright (c) 2008, http://www.snakeyaml.org + +The binary distribution of this product bundles binaries of +swagger-annotations (https://github.com/swagger-api/swagger-core), +which has the following notices: + * Copyright 2016 SmartBear Software http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344e8ae/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 8aeeabd..8b3d292 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -98,10 +98,6 @@ etc/hadoop - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version} - /share/hadoop/${hadoop.component}/lib/services - - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target /share/hadoop/${hadoop.component}/sources @@ -109,10 +105,6 @@ - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version} - /share/hadoop/${hadoop.component}/lib/services-api - - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target /share/hadoop/${hadoop.component}/sources http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344e8ae/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
[65/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java new file mode 100644 index 000..3e53418 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.UsageException; +import org.apache.hadoop.yarn.service.api.records.ConfigFormat; + +import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY; +import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; +import java.io.File; + +/** + * Registry actions + * + * --instance {app name}, if a / is in it, refers underneath? + * --dest {destfile} + * --list : list instances of slider service + * --listfiles + */ +@Parameters(commandNames = {ACTION_REGISTRY}, +commandDescription = DESCRIBE_ACTION_REGISTRY) + +public class ActionRegistryArgs extends AbstractActionArgs { + + public static final String USAGE = + "Usage: " + SliderActions.ACTION_REGISTRY + + " (" + + Arguments.ARG_LIST + "|" + + Arguments.ARG_LISTCONF + "|" + + Arguments.ARG_LISTEXP + "|" + + Arguments.ARG_LISTFILES + "|" + + Arguments.ARG_GETCONF + "|" + + Arguments.ARG_GETEXP + "> " + + Arguments.ARG_NAME + " " + + " )" + + "[" + Arguments.ARG_VERBOSE + "] " + + "[" + Arguments.ARG_USER + "] " + + "[" + Arguments.ARG_OUTPUT + " ] " + + "[" + Arguments.ARG_SERVICETYPE + " ] " + + "[" + Arguments.ARG_FORMAT + "] " + + System.getProperty("line.separator") + + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" + ; + public ActionRegistryArgs() { + } + + public ActionRegistryArgs(String name) { +this.name = name; + } + + @Override + public String getActionName() { +return ACTION_REGISTRY; + } + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + @Override + public int getMinParams() { +return 0; + } + + @Parameter(names = {ARG_LIST}, + description = "list services") + public boolean list; + + @Parameter(names = {ARG_LISTCONF}, + description = "list configurations") + public boolean listConf; + + @Parameter(names = {ARG_GETCONF}, + description = "get configuration") + public String getConf; + + @Parameter(names = {ARG_LISTEXP}, + description = "list exports") + public boolean listExports; + + @Parameter(names = {ARG_GETEXP}, + description = "get export") + public String getExport; + + @Parameter(names = {ARG_LISTFILES}, + description = "list files") + public String listFiles; + + @Parameter(names = {ARG_GETFILES}, + description = "get files") + public String getFiles; + + //--format + @Parameter(names = ARG_FORMAT, + description = "Format for a response: ") + public String format = ConfigFormat.XML.toString() ; + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST}, + description = "Output destination") + public File out; + +
[55/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java deleted file mode 100644 index e10305a..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.api.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import java.util.Locale; - -@InterfaceAudience.Public -@InterfaceStability.Unstable -public enum ConfigFormat { - - JSON("json"), - PROPERTIES("properties"), - XML("xml"), - HADOOP_XML("hadoop_xml"), - ENV("env"), - TEMPLATE("template"), - YAML("yaml"), - ; - ConfigFormat(String suffix) { -this.suffix = suffix; - } - - private final String suffix; - - public String getSuffix() { -return suffix; - } - - - @Override - public String toString() { -return suffix; - } - - /** - * Get a matching format or null - * @param type - * @return the format - */ - public static ConfigFormat resolve(String type) { -for (ConfigFormat format: values()) { - if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) { -return format; - } -} -return null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java deleted file mode 100644 index 0ac508b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.api.records; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import
[66/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java new file mode 100644 index 000..c3a2752 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -0,0 +1,892 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; +import org.apache.hadoop.yarn.service.ClientAMProtocol; +import org.apache.hadoop.yarn.service.ServiceMaster; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; +import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; +import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import
[26/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java deleted file mode 100644 index 86025ee..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import org.apache.hadoop.util.VersionInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Locale; -import java.util.Properties; - -/** - * Extract the version properties, which will look something like - * - * application.name=${pom.name} - * application.version=${pom.version} - * application.build=${buildNumber} - * application.build.java.version=${java.version} - * application.build.info=${pom.name}-${pom.version} Built against ${buildNumber} on ${java.version} by ${user.name} - * - * - * the mvn process-resources target will expand the properties - * and add the resources to target/classes, which will then look something like - * - * application.name=Slider Core - * application.version=0.7.1-SNAPSHOT - * application.build=1dd69 - * application.build.java.version=1.7.0_45 - * application.build.user=stevel - * application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 1.7.0_45 by stevel - * - * - * Note: the values will change and more properties added. - */ -public class SliderVersionInfo { - private static final Logger log = LoggerFactory.getLogger(SliderVersionInfo.class); - - /** - * Name of the resource containing the filled-in-at-runtime props - */ - public static final String VERSION_RESOURCE = - "org/apache/slider/providers/dynamic/application.properties"; - - public static final String APP_NAME = "application.name"; - public static final String APP_VERSION = "application.version"; - public static final String APP_BUILD = "application.build"; - public static final String APP_BUILD_JAVA_VERSION = "application.build.java.version"; - public static final String APP_BUILD_USER = "application.build.user"; - public static final String APP_BUILD_INFO = "application.build.info"; - public static final String HADOOP_BUILD_INFO = "hadoop.build.info"; - public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info"; - - - public static Properties loadVersionProperties() { -Properties props = new Properties(); -URL resURL = SliderVersionInfo.class.getClassLoader() - .getResource(VERSION_RESOURCE); -assert resURL != null : "Null resource " + VERSION_RESOURCE; - -try { - InputStream inStream = resURL.openStream(); - assert inStream != null : "Null input stream from " + VERSION_RESOURCE; - props.load(inStream); -} catch (IOException e) { - log.warn("IOE loading " + VERSION_RESOURCE, e); -} -return props; - } - - /** - * Load the version info and print it - * @param logger logger - */ - public static void loadAndPrintVersionInfo(Logger logger) { -Properties props = loadVersionProperties(); -logger.info(props.getProperty(APP_BUILD_INFO)); -logger.info("Compiled against Hadoop {}", -props.getProperty(HADOOP_BUILD_INFO)); -logger.info(getHadoopVersionString()); - } - - public static String getHadoopVersionString() { -return String.format(Locale.ENGLISH, -"Hadoop runtime version %s with source checksum %s and build date %s", -VersionInfo.getBranch(), -
[31/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java deleted file mode 100644 index 7712191..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ /dev/null @@ -1,2783 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.slider.client; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathNotFoundException; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.exceptions.NoRecordException; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.RegistryPathStatus; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.KerberosDiags; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ApplicationTimeout; -import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.util.Records; -import org.apache.hadoop.yarn.util.Times; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; -import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.api.proto.Messages; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.client.ipc.SliderClusterOperations; -import org.apache.slider.common.Constants; -import
[36/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java new file mode 100644 index 000..c0712c3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SaslPropertiesResolver; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.regex.Pattern; + +import static org.apache.hadoop.security.UserGroupInformation.*; +import static org.apache.hadoop.security.authentication.util.KerberosUtil.*; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; + +/** + * Kerberos diagnostics + * At some point this may move to hadoop core, so please keep use of slider + * methods and classes to ~0. + * + * This operation expands some of the diagnostic output of the security code, + * but not all. For completeness + * + * Set the environment variable {@code HADOOP_JAAS_DEBUG=true} + * Set the log level for {@code org.apache.hadoop.security=DEBUG} + */ +public class KerberosDiags implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class); + public static final String KRB5_CCNAME = "KRB5CCNAME"; + public static final String JAVA_SECURITY_KRB5_CONF += "java.security.krb5.conf"; + public static final String JAVA_SECURITY_KRB5_REALM += "java.security.krb5.realm"; + public static final String SUN_SECURITY_KRB5_DEBUG += "sun.security.krb5.debug"; + public static final String SUN_SECURITY_SPNEGO_DEBUG += "sun.security.spnego.debug"; + public static final String SUN_SECURITY_JAAS_FILE += "java.security.auth.login.config"; + public static final String KERBEROS_KINIT_COMMAND += "hadoop.kerberos.kinit.command"; + public static final String HADOOP_AUTHENTICATION_IS_DISABLED + = "Hadoop authentication is disabled"; + public static final String UNSET = "(unset)"; + public static final String NO_DEFAULT_REALM = "Cannot locate default realm"; + + private final Configuration conf; + private final List services; + private final PrintStream out; + private final File keytab; + private final String principal; + private final long minKeyLength; + private final boolean securityRequired; + + public static final String CAT_JVM = "JVM"; + public static final String CAT_JAAS = "JAAS"; + public static final String CAT_CONFIG = "CONFIG"; + public static final String CAT_LOGIN = "LOGIN"; + public static final String CAT_KERBEROS = "KERBEROS"; + public static
[54/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java deleted file mode 100644 index 1049698..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ /dev/null @@ -1,872 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client; - -import org.apache.commons.lang.StringUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryNTimes; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ApplicationTimeout; -import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; -import org.apache.hadoop.yarn.service.ClientAMProtocol; -import org.apache.hadoop.yarn.service.ServiceMaster; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; -import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; -import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; -import org.apache.hadoop.yarn.service.client.params.Arguments; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; -import org.apache.hadoop.yarn.service.client.params.CommonArgs; -import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
[53/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java deleted file mode 100644 index c2866cf..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.UsageException; -import org.apache.hadoop.yarn.service.api.records.ConfigFormat; - -import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY; -import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; -import java.io.File; - -/** - * Registry actions - * - * --instance {app name}, if a / is in it, refers underneath? - * --dest {destfile} - * --list : list instances of slider service - * --listfiles - */ -@Parameters(commandNames = {ACTION_REGISTRY}, -commandDescription = DESCRIBE_ACTION_REGISTRY) - -public class ActionRegistryArgs extends AbstractActionArgs { - - public static final String USAGE = - "Usage: " + SliderActions.ACTION_REGISTRY - + " (" - + Arguments.ARG_LIST + "|" - + Arguments.ARG_LISTCONF + "|" - + Arguments.ARG_LISTEXP + "|" - + Arguments.ARG_LISTFILES + "|" - + Arguments.ARG_GETCONF + "|" - + Arguments.ARG_GETEXP + "> " - + Arguments.ARG_NAME + " " - + " )" - + "[" + Arguments.ARG_VERBOSE + "] " - + "[" + Arguments.ARG_USER + "] " - + "[" + Arguments.ARG_OUTPUT + " ] " - + "[" + Arguments.ARG_SERVICETYPE + " ] " - + "[" + Arguments.ARG_FORMAT + "] " - + System.getProperty("line.separator") - + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" - ; - public ActionRegistryArgs() { - } - - public ActionRegistryArgs(String name) { -this.name = name; - } - - @Override - public String getActionName() { -return ACTION_REGISTRY; - } - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - @Override - public int getMinParams() { -return 0; - } - - @Parameter(names = {ARG_LIST}, - description = "list services") - public boolean list; - - @Parameter(names = {ARG_LISTCONF}, - description = "list configurations") - public boolean listConf; - - @Parameter(names = {ARG_GETCONF}, - description = "get configuration") - public String getConf; - - @Parameter(names = {ARG_LISTEXP}, - description = "list exports") - public boolean listExports; - - @Parameter(names = {ARG_GETEXP}, - description = "get export") - public String getExport; - - @Parameter(names = {ARG_LISTFILES}, - description = "list files") - public String listFiles; - - @Parameter(names = {ARG_GETFILES}, - description = "get files") - public String getFiles; - - //--format - @Parameter(names = ARG_FORMAT, - description = "Format for a response: ") - public String format = ConfigFormat.XML.toString() ; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST}, - description = "Output destination") - public File out; - - @Parameter(names
[41/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 0ed4860..1049698 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -22,6 +22,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryNTimes; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,6 +42,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -58,34 +61,32 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.ServiceMaster; +import org.apache.hadoop.yarn.service.api.records.Application; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; import org.apache.hadoop.yarn.service.client.params.Arguments; import org.apache.hadoop.yarn.service.client.params.ClientArgs; import org.apache.hadoop.yarn.service.client.params.CommonArgs; import org.apache.hadoop.yarn.service.conf.SliderExitCodes; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.UsageException; -import org.apache.slider.core.launch.ClasspathConstructor; -import org.apache.slider.core.launch.JavaCommandLineBuilder; -import org.apache.slider.core.registry.SliderRegistryUtils; -import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.core.zk.ZookeeperUtils; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.exceptions.UsageException; +import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; +import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder; +import
[15/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java deleted file mode 100644 index 15333e4..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ /dev/null @@ -1,1123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.state; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.server.appmaster.management.BoolMetric; -import org.apache.slider.server.appmaster.management.Timestamp; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; -import org.apache.slider.server.avro.LoadedRoleHistory; -import org.apache.slider.server.avro.NodeEntryRecord; -import org.apache.slider.server.avro.RoleHistoryHeader; -import org.apache.slider.server.avro.RoleHistoryWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; - -/** - * The Role History. - * - * Synchronization policy: all public operations are synchronized. - * Protected methods are in place for testing -no guarantees are made. - * - * Inner classes have no synchronization guarantees; they should be manipulated - * in these classes and not externally. - * - * Note that as well as some methods marked visible for testing, there - * is the option for the time generator method, {@link #now()} to - * be overridden so that a repeatable time series can be used. - * - */ -public class RoleHistory { - protected static final Logger log = -LoggerFactory.getLogger(RoleHistory.class); - private final List providerRoles; - /** the roles in here are shared with App State */ - private final MaproleStatusMap = new HashMap<>(); - private final AbstractClusterServices recordFactory; - - private long startTime; - - /** Time when saved */ - private final Timestamp saveTime = new Timestamp(0); - - /** If the history was loaded, the time at which the history was saved. - * That is: the time the data was valid */ - private final Timestamp thawedDataTime = new Timestamp(0); - - private NodeMap nodemap; - private int roleSize; - private final BoolMetric dirty = new BoolMetric(false); - private FileSystem filesystem; - private Path historyPath; - private RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - - /** - * When were the nodes updated in a {@link #onNodesUpdated(List)} call? - * If zero: never. - */ - private final Timestamp nodesUpdatedTime = new Timestamp(0); - private final BoolMetric nodeUpdateReceived = new BoolMetric(false); - - private OutstandingRequestTracker
[38/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 647bfe9..a044838 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -24,30 +24,24 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.Configuration; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.AbstractLauncher; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; import org.apache.hadoop.yarn.service.ServiceContext; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.hadoop.yarn.service.api.records.Application; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.api.records.ConfigFormat; +import org.apache.hadoop.yarn.service.api.records.Configuration; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.conf.YarnServiceConf; +import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.utils.PublishedConfiguration; +import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,23 +49,18 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; -import static org.apache.slider.api.ServiceApiConstants.*; -import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$; +import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*; /** * This is a factoring out of methods handy for providers. It's bonded to a log * at construction time. */ -public class ProviderUtils implements RoleKeys, SliderKeys { +public class ProviderUtils implements YarnServiceConstants { protected static final Logger log = LoggerFactory.getLogger(ProviderUtils.class); @@ -174,46 +163,22 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } /** - * Get resource requirements from a String value. If value isn't specified, - * use the default value. If value is greater than max, use the max value. - * @param val string value - * @param defVal default value - * @param maxVal maximum value
[42/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java new file mode 100644 index 000..b57225a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.Objects; + +import javax.xml.bind.annotation.XmlRootElement; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The current status of a submitted application, returned as a response to the + * GET API. + **/ +@InterfaceAudience.Public +@InterfaceStability.Unstable +@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.") +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +@XmlRootElement +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ApplicationStatus extends BaseResource { + private static final long serialVersionUID = -3469885905347851034L; + + private String diagnostics = null; + private ApplicationState state = null; + private Integer code = null; + + /** + * Diagnostic information (if any) for the reason of the current state of the + * application. It typically has a non-null value, if the application is in a + * non-running state. + **/ + public ApplicationStatus diagnostics(String diagnostics) { +this.diagnostics = diagnostics; +return this; + } + + @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.") + @JsonProperty("diagnostics") + public String getDiagnostics() { +return diagnostics; + } + + public void setDiagnostics(String diagnostics) { +this.diagnostics = diagnostics; + } + + /** + * Application state. + **/ + public ApplicationStatus state(ApplicationState state) { +this.state = state; +return this; + } + + @ApiModelProperty(example = "null", value = "Application state.") + @JsonProperty("state") + public ApplicationState getState() { +return state; + } + + public void setState(ApplicationState state) { +this.state = state; + } + + /** + * An error code specific to a scenario which app owners should be able to use + * to understand the failure in addition to the diagnostic information. + **/ + public ApplicationStatus code(Integer code) { +this.code = code; +return this; + } + + @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.") + @JsonProperty("code") + public Integer getCode() { +return code; + } + + public void setCode(Integer code) { +this.code = code; + } + + @Override + public boolean equals(java.lang.Object o) { +if (this == o) { + return true; +} +if (o == null || getClass() != o.getClass()) { + return false; +} +ApplicationStatus applicationStatus = (ApplicationStatus) o; +return Objects.equals(this.diagnostics,
[11/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java deleted file mode 100644 index 9c653f3..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.services.workflow; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.service.ServiceStateChangeListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * An extended composite service which stops itself if any child service - * fails, or when all its children have successfully stopped without failure. - * - * Lifecycle - * - * If any child exits with a failure: this service stops, propagating - * the exception. - * When all child services has stopped, this service stops itself - * - * - */ -public class WorkflowCompositeService extends CompositeService -implements ServiceParent, ServiceStateChangeListener { - - private static final Logger LOG = -LoggerFactory.getLogger(WorkflowCompositeService.class); - - /** - * Deadlock-avoiding overridden config for slider services; see SLIDER-1052 - */ - private volatile Configuration configuration; - - /** - * Construct an instance - * @param name name of this service instance - */ - public WorkflowCompositeService(String name) { -super(name); - } - - @Override - public Configuration getConfig() { -return configuration; - } - - @Override - protected void setConfig(Configuration conf) { -super.setConfig(conf); -configuration = conf; - } - - /** - * Construct an instance with the default name. - */ - public WorkflowCompositeService() { -this("WorkflowCompositeService"); - } - - /** - * Varargs constructor - * @param name name of this service instance - * @param children children - */ - public WorkflowCompositeService(String name, Service... children) { -this(name); -for (Service child : children) { - addService(child); -} - } - - /** - * Construct with a list of children - * @param name name of this service instance - * @param children children to add - */ - public WorkflowCompositeService(String name, List children) { -this(name); -for (Service child : children) { - addService(child); -} - } - - /** - * Add a service, and register it - * @param service the {@link Service} to be added. - * Important: do not add a service to a parent during your own serviceInit/start, - * in Hadoop 2.2; you will trigger a ConcurrentModificationException. - */ - @Override - public synchronized void addService(Service service) { -Preconditions.checkArgument(service != null, "null service argument"); -service.registerServiceListener(this); -super.addService(service); - } - - /** - * When this service is started, any service stopping with a failure - * exception is converted immediately into a failure of this service, - * storing the failure and stopping ourselves. - * @param child the service that has changed. - */ - @Override - public void stateChanged(Service child) { -//if that child stopped while we are running: -if (isInState(STATE.STARTED) && child.isInState(STATE.STOPPED)) { - // a child service
[43/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties deleted file mode 100644 index 333859e..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is the log4j configuration for Slider Application Master - -# Log rotation based on size (100KB) with a max of 10 backup files -log4j.rootLogger=INFO, amlog -log4j.threshhold=ALL -log4j.appender.amlog=org.apache.log4j.RollingFileAppender -log4j.appender.amlog.layout=org.apache.log4j.PatternLayout -log4j.appender.amlog.File=${LOG_DIR}/slider.log -log4j.appender.amlog.MaxFileSize=1MB -log4j.appender.amlog.MaxBackupIndex=10 - -# log layout skips stack-trace creation operations by avoiding line numbers and method -log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -# debug edition is much more expensive -#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n - -# configure stderr -# set the conversion pattern of stderr -# Print the date in ISO 8601 format -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.Target=System.err -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -log4j.appender.subprocess=org.apache.log4j.ConsoleAppender -log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout -log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n -#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess - -# for debugging Slider -#log4j.logger.org.apache.slider=DEBUG - -# uncomment to debug service lifecycle issues -#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG -#log4j.logger.org.apache.hadoop.yarn.service=DEBUG - -# uncomment for YARN operations -#log4j.logger.org.apache.hadoop.yarn.client=DEBUG - -# uncomment this to debug security problems -#log4j.logger.org.apache.hadoop.security=DEBUG - -#crank back on some noise -log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -log4j.logger.org.apache.hadoop.hdfs=WARN -log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR - -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.curator.framework.state=ERROR -log4j.logger.org.apache.curator.framework.imps=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties new file mode 100644 index 000..58c8e27 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in
[59/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java new file mode 100644 index 000..6e6f4dd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java @@ -0,0 +1,1654 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.GlobFilter; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import org.apache.hadoop.yarn.service.client.params.SliderActions; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.zookeeper.server.util.KerberosUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.Serializable; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.net.URLDecoder; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * These are slider-specific Util methods + */ +public final class SliderUtils { + + private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); + + /**
[08/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java deleted file mode 100644 index 057f6c5..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.common.tools; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** Test slider util methods. */ -public class TestSliderUtils { - protected static final Logger log = - LoggerFactory.getLogger(TestSliderUtils.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - //@Test - public void testTruncate() { -Assert.assertEquals(SliderUtils.truncate(null, 5), null); -Assert.assertEquals(SliderUtils.truncate("323", -1), "323"); -Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232"); -Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890"); -Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), "123456789012345"); -Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), "12345678901..."); -Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1"); -Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890"); -Assert.assertEquals(SliderUtils.truncate("", 10), ""); - } - - //@Test - public void testApplicationReportComparison() { -List instances = getApplicationReports(); - -SliderUtils.sortApplicationsByMostRecent(instances); - -Assert.assertEquals(1000, instances.get(0).getStartTime()); -Assert.assertEquals(1000, instances.get(1).getStartTime()); -Assert.assertEquals(1000, instances.get(2).getStartTime()); -Assert.assertEquals(1000, instances.get(3).getStartTime()); - -instances = getApplicationReports(); - -SliderUtils.sortApplicationReport(instances); -Assert.assertEquals(1000, instances.get(0).getStartTime()); -Assert.assertEquals(1000, instances.get(1).getStartTime()); -Assert.assertEquals(1000, instances.get(2).getStartTime()); -Assert.assertEquals(1000, instances.get(3).getStartTime()); - -Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING); -Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING); -Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED || - instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING); -Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED); - } - - private List getApplicationReports() { -List instances = new ArrayList(); -instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED)); -instances.add(getApplicationReport(900, 998, "app1",
[14/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java deleted file mode 100644 index 0cac430..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.health.HealthCheckRegistry; -import com.codahale.metrics.servlets.HealthCheckServlet; -import com.codahale.metrics.servlets.MetricsServlet; -import com.codahale.metrics.servlets.PingServlet; -import com.codahale.metrics.servlets.ThreadDumpServlet; -import com.google.common.base.Preconditions; -import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter; -import com.sun.jersey.api.core.ResourceConfig; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.spi.container.servlet.ServletContainer; -import org.apache.hadoop.yarn.webapp.Dispatcher; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.apache.hadoop.yarn.webapp.WebApp; -import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig; -import org.apache.slider.server.appmaster.web.rest.AMWebServices; -import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider; - -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; - -/** - * - */ -public class SliderAMWebApp extends WebApp { - public static final String BASE_PATH = "slideram"; - public static final String CONTAINER_STATS = "/stats"; - public static final String CLUSTER_SPEC = "/spec"; - - private final WebAppApi webAppApi; - - public SliderAMWebApp(WebAppApi webAppApi) { -Preconditions.checkArgument(webAppApi != null, "webAppApi null"); -this.webAppApi = webAppApi; - } - - @Override - public void setup() { -Logger.getLogger("com.sun.jersey").setLevel(Level.FINEST); -// Make one of these to ensure that the jax-b annotations -// are properly picked up. -bind(SliderJacksonJaxbJsonProvider.class); - -// Get exceptions printed to the screen -bind(GenericExceptionHandler.class); -// bind the REST interface -bind(AMWebServices.class); - -//bind(AMAgentWebServices.class); -route("/", SliderAMController.class); -route(CONTAINER_STATS, SliderAMController.class, "containerStats"); -route(CLUSTER_SPEC, SliderAMController.class, "specification"); - } - - @Override - public void configureServlets() { -setup(); - -serve("/", "/__stop").with(Dispatcher.class); - -for (String path : this.getServePathSpecs()) { - serve(path).with(Dispatcher.class); -} - -serve(SYSTEM_HEALTHCHECK) -.with(new HealthCheckServlet(new HealthCheckRegistry())); -serve(SYSTEM_METRICS).with(new MetricsServlet(new MetricRegistry())); -serve(SYSTEM_PING).with(new PingServlet()); -serve(SYSTEM_THREADS).with(new ThreadDumpServlet()); - -String regex = "(?!/ws)"; -serveRegex(regex).with(SliderDefaultWrapperServlet.class); - -Mapparams = new HashMap<>(); -params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true"); -params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true"); -params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true"); -params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS,
[60/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java new file mode 100644 index 000..7b22e3e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.codehaus.jackson.map.SerializationConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Support for marshalling objects to and from JSON. + * This class is NOT thread safe; it constructs an object mapper + * as an instance field. + * @param + */ +public class JsonSerDeser { + + private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class); + private static final String UTF_8 = "UTF-8"; + + private final Class classType; + private final ObjectMapper mapper; + + /** + * Create an instance bound to a specific type + * @param classType class type + */ + public JsonSerDeser(Class classType) { +this.classType = classType; +this.mapper = new ObjectMapper(); +mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public JsonSerDeser(Class classType, PropertyNamingStrategy namingStrategy) { +this(classType); +mapper.setPropertyNamingStrategy(namingStrategy); + } + + /** + * Convert from JSON + * @param json input + * @return the parsed JSON + * @throws IOException IO + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromJson(String json) +throws IOException, JsonParseException, JsonMappingException { +try { + return mapper.readValue(json, classType); +} catch (IOException e) { + log.error("Exception while parsing json : " + e + "\n" + json, e); + throw e; +} + } + + /** + * Convert from a JSON file + * @param jsonFile input file + * @return the parsed JSON + * @throws IOException IO problems + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromFile(File jsonFile) +throws IOException, JsonParseException, JsonMappingException { +File absoluteFile = jsonFile.getAbsoluteFile(); +try { + return mapper.readValue(absoluteFile, classType); +} catch (IOException e) { + log.error("Exception while parsing json file {}", absoluteFile, e); + throw e; +} + } + + /** + * Convert from a JSON file + * @param resource input file + * @return the parsed JSON + * @throws IOException IO problems + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromResource(String resource) +throws IOException, JsonParseException, JsonMappingException { +
[32/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java deleted file mode 100644 index b28257f..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.client; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.PathNotFoundException; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.exceptions.NoRecordException; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.NotFoundException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.PublishedConfigSet; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.registry.retrieve.RegistryRetriever; - -import java.io.File; -import java.io.IOException; - -import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser; -import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath; - -public class ClientUtils { - public static ServiceRecord lookupServiceRecord(RegistryOperations rops, - String user, String name) throws IOException, SliderException { -return lookupServiceRecord(rops, user, null, name); - } - - public static ServiceRecord lookupServiceRecord(RegistryOperations rops, - String user, String type, String name) throws IOException, - SliderException { -if (StringUtils.isEmpty(user)) { - user = currentUser(); -} else { - user = RegistryPathUtils.encodeForRegistry(user); -} -if (StringUtils.isEmpty(type)) { - type = SliderKeys.APP_TYPE; -} - -String path = servicePath(user, type, name); -return resolve(rops, path); - } - - public static ServiceRecord resolve(RegistryOperations rops, String path) - throws IOException, SliderException { -try { - return rops.resolve(path); -} catch (PathNotFoundException | NoRecordException e) { - throw new NotFoundException(e.getPath().toString(), e); -} - } - - public static PublishedConfiguration getConfigFromRegistry( - RegistryOperations rops, Configuration configuration, - String configName, String appName, String user, boolean external) - throws IOException, SliderException { -ServiceRecord instance = lookupServiceRecord(rops, user, appName); - -RegistryRetriever retriever = new RegistryRetriever(configuration, instance); -PublishedConfigSet configurations = retriever.getConfigurations(external); - -PublishedConfiguration published = retriever.retrieveConfiguration( -configurations, configName, external); -return published; - } - - public static String saveOrReturnConfig(PublishedConfiguration published, - String format, File destPath, String fileName) - throws BadCommandArgumentsException, IOException { -ConfigFormat configFormat = ConfigFormat.resolve(format); -if (configFormat == null) { - throw new BadCommandArgumentsException( - "Unknown/Unsupported format %s
[09/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java deleted file mode 100644 index 7b0586e..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.client; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.client.params.Arguments; -import org.apache.hadoop.yarn.service.client.params.SliderActions; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.UsageException; -import org.apache.slider.utils.SliderTestBase; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; - -/** - * Test the argument parsing/validation logic. - */ -public class TestClientBadArgs extends SliderTestBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestClientBadArgs.class); - - //@Test - public void testNoAction() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - "Usage: slider COMMAND", - EMPTY_LIST); - - } - - //@Test - public void testUnknownAction() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - "not-a-known-action", - Arrays.asList("not-a-known-action")); - } - - //@Test - public void testActionWithoutOptions() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - "Usage: slider build ", - Arrays.asList(SliderActions.ACTION_BUILD)); - } - - //@Test - public void testActionWithoutEnoughArgs() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS, - Arrays.asList(SliderActions.ACTION_START)); - } - - //@Test - public void testActionWithTooManyArgs() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - ErrorStrings.ERROR_TOO_MANY_ARGUMENTS, - Arrays.asList(SliderActions.ACTION_HELP, - "hello, world")); - } - - //@Test - public void testBadImageArg() throws Throwable { -launchExpectingException(SliderClient.class, - createTestConfig(), - "Unknown option: --image", -Arrays.asList(SliderActions.ACTION_HELP, - Arguments.ARG_IMAGE)); - } - - //@Test - public void testRegistryUsage() throws Throwable { -Throwable exception = launchExpectingException(SliderClient.class, -createTestConfig(), -"org.apache.slider.core.exceptions.UsageException: Argument --name " + -"missing", -Arrays.asList(SliderActions.ACTION_REGISTRY)); -assertTrue(exception instanceof UsageException); -LOG.info(exception.toString()); - } - - //@Test - public void testRegistryExportBadUsage1() throws Throwable { -
[35/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java new file mode 100644 index 000..415392a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java @@ -0,0 +1,1699 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.GlobFilter; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import org.apache.hadoop.yarn.service.client.params.SliderActions; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.zookeeper.server.util.KerberosUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.Serializable; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.net.URLDecoder; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * These are slider-specific Util methods + */ +public final class SliderUtils { + + private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); + + /** + * Atomic
[28/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java deleted file mode 100644 index 64fd8ae..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ /dev/null @@ -1,611 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.yarn.service.conf.SliderKeys; -import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; -import org.apache.slider.core.exceptions.BadConfigException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -/** - * Methods to aid in config, both in the Configuration class and - * with other parts of setting up Slider-initated processes. - * - * Some of the methods take an argument of a map iterable for their sources; this allows - * the same method - */ -public class ConfigHelper { - private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); - - /** - * Dump the (sorted) configuration - * @param conf config - * @return the sorted keyset - */ - public static Set dumpConf(Configuration conf) { -Set keys = sortedConfigKeys(conf); -for (String key : keys) { - log.info("{}={}", key, conf.get(key)); -} -return keys; - } - - /** - * Take a configuration and return a sorted set - * @param conf config - * @return the sorted keyset - - */ - public static Set sortedConfigKeys(Iterable> conf) { -TreeSet sorted = new TreeSet(); -for (Map.Entry entry : conf) { - sorted.add(entry.getKey()); -} -return sorted; - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Map map, - String origin) throws BadConfigException { -addConfigMap(config, map.entrySet(), origin); - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Iterable > map, - String origin) throws BadConfigException { -for (Map.Entry mapEntry : map) { - String key = mapEntry.getKey(); - String value = mapEntry.getValue(); - if (value == null) { -throw new BadConfigException("Null value for property " + key); - } -
[56/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java deleted file mode 100644 index 8c968dc..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ /dev/null @@ -1,655 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.UpdatedContainer; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.TimelineV2Client; -import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; -import org.apache.hadoop.yarn.client.api.async.NMClientAsync; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.ConfigFile; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType; -import org.apache.hadoop.yarn.service.component.Component; -import org.apache.hadoop.yarn.service.component.ComponentEvent; -import org.apache.hadoop.yarn.service.component.ComponentEventType; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; -import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; -import org.apache.hadoop.yarn.service.provider.ProviderUtils; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; -import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; -import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; -import
[06/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java deleted file mode 100644 index 8686479..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority; -import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole; - -/** - * Test app state RM operations. - */ -public class TestMockAppStateRMOperations extends BaseMockAppStateTest -implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(BaseMockAppStateTest.class); - - @Override - public String getTestName() { -return "TestMockAppStateRMOperations"; - } - - //@Test - public void testPriorityOnly() throws Throwable { -assertEquals(5, extractRole(buildPriority(5, false))); - } - - //@Test - public void testPriorityRoundTrip() throws Throwable { -assertEquals(5, extractRole(buildPriority(5, false))); - } - - //@Test - public void testPriorityRoundTripWithRequest() throws Throwable { -int priority = buildPriority(5, false); -assertEquals(5, extractRole(priority)); - } - - //@Test - public void testMockAddOp() throws Throwable { -getRole0Status().setDesired(1); -List ops = appState.reviewRequestAndReleaseNodes(); -assertListLength(ops, 1); -ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0); -int priority = operation.getRequest().getPriority().getPriority(); -assertEquals(extractRole(priority), getRole0Status().getKey()); -MockRMOperationHandler handler = new MockRMOperationHandler(); -handler.execute(ops); - -AbstractRMOperation op = handler.getFirstOp(); -assertTrue(op instanceof ContainerRequestOperation); - } - - /** - * Test of a flex up and down op which verifies that outstanding - * requests are cancelled first. - * - * request 5 nodes, assert 5 request made - * allocate 1 of them - * flex cluster size to 3 - * assert this generates 2 cancel requests - * - */ - //@Test - public void testRequestThenCancelOps()
[46/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java deleted file mode 100644 index 1fa07ce..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.net.HostAndPort; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; - -import java.util.ArrayList; -import java.util.List; - -public class ZookeeperUtils { - public static final int DEFAULT_PORT = 2181; - - public static String buildConnectionString(String zkHosts, int port) { -String zkPort = Integer.toString(port); -//parse the hosts -String[] hostlist = zkHosts.split(",", 0); -String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false); -return quorum; - } - - /** - * Take a quorum list and split it to (trimmed) pairs - * @param hostPortQuorumList list of form h1:port, h2:port2,... - * @return a possibly empty list of values between commas. They may not be - * valid hostname:port pairs - */ - public static List splitToPairs(String hostPortQuorumList) { -// split an address hot -String[] strings = StringUtils.getStrings(hostPortQuorumList); -int len = 0; -if (strings != null) { - len = strings.length; -} -List tuples = new ArrayList(len); -if (strings != null) { - for (String s : strings) { -tuples.add(s.trim()); - } -} -return tuples; - } - - /** - * Split a quorum list into a list of hostnames and ports - * @param hostPortQuorumList split to a list of hosts and ports - * @return a list of values - */ - public static List splitToHostsAndPorts(String hostPortQuorumList) { -// split an address hot -String[] strings = StringUtils.getStrings(hostPortQuorumList); -int len = 0; -if (strings != null) { - len = strings.length; -} -List list = new ArrayList(len); -if (strings != null) { - for (String s : strings) { - list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT)); - } -} -return list; - } - - /** - * Build up to a hosts only list - * @param hostAndPorts - * @return a list of the hosts only - */ - public static String buildHostsOnlyList(List hostAndPorts) { -StringBuilder sb = new StringBuilder(); -for (HostAndPort hostAndPort : hostAndPorts) { - sb.append(hostAndPort.getHostText()).append(","); -} -if (sb.length() > 0) { - sb.delete(sb.length() - 1, sb.length()); -} -return sb.toString(); - } - - public static String buildQuorumEntry(HostAndPort hostAndPort, -int defaultPort) { -String s = hostAndPort.toString(); -if (hostAndPort.hasPort()) { - return s; -} else { - return s + ":" + defaultPort; -} - } - - /** - * Build a quorum list, injecting a ":defaultPort" ref if needed on - * any entry without one - * @param hostAndPorts - * @param defaultPort - * @return - */ - public static String buildQuorum(List hostAndPorts, int defaultPort) { -List entries = new ArrayList(hostAndPorts.size()); -for (HostAndPort hostAndPort : hostAndPorts) { - entries.add(buildQuorumEntry(hostAndPort, defaultPort)); -} -return SliderUtils.join(entries, ",", false); - } - - public static String convertToHostsOnlyList(String quorum) throws - BadConfigException { -List
[44/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b3aff2f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b3aff2f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b3aff2f Branch: refs/heads/yarn-native-services Commit: 2b3aff2fb0d71e85787360f2aa1c422c474b24ca Parents: a8ad52a Author: Billie RinaldiAuthored: Wed Aug 23 12:31:47 2017 -0700 Committer: Jian He Committed: Wed Aug 30 22:22:30 2017 -0700 -- .../assemblies/hadoop-yarn-services-dist.xml| 30 + .../assemblies/hadoop-yarn-slider-dist.xml | 30 - hadoop-yarn-project/hadoop-yarn/bin/yarn|8 +- .../service/webapp/ApplicationApiService.java | 275 ++ .../service/webapp/ApplicationApiWebApp.java| 123 + .../yarn/services/api/ApplicationApi.java | 36 - .../api/impl/ApplicationApiService.java | 288 -- .../services/webapp/ApplicationApiWebApp.java | 125 - ...RN-Simplified-V1-API-Layer-For-Services.yaml |3 - .../src/main/webapp/WEB-INF/web.xml |2 +- .../dev-support/findbugs-exclude.xml| 93 +- .../conf/slideram-log4j.properties | 68 - .../conf/yarnservice-log4j.properties | 62 + .../hadoop-yarn-slider-core/pom.xml | 30 +- .../slider/server/avro/RoleHistoryRecord.avsc | 114 - .../apache/hadoop/security/KerberosDiags.java | 677 - .../yarn/service/ContainerFailureTracker.java |4 +- .../yarn/service/ContainerLaunchService.java| 101 - .../hadoop/yarn/service/ServiceContext.java |6 +- .../hadoop/yarn/service/ServiceMaster.java |9 +- .../hadoop/yarn/service/ServiceMonitor.java | 149 - .../hadoop/yarn/service/ServiceScheduler.java | 44 +- .../api/constants/ServiceApiConstants.java | 69 + .../yarn/service/api/records/Application.java | 466 +++ .../service/api/records/ApplicationState.java | 33 + .../service/api/records/ApplicationStatus.java | 148 + .../yarn/service/api/records/Artifact.java | 160 + .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 412 +++ .../yarn/service/api/records/ConfigFile.java| 225 ++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 ++ .../yarn/service/api/records/Container.java | 297 ++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 + .../service/api/records/PlacementPolicy.java| 102 + .../service/api/records/ReadinessCheck.java | 175 ++ .../yarn/service/api/records/Resource.java | 159 + .../hadoop/yarn/service/client/ServiceCLI.java | 14 +- .../yarn/service/client/ServiceClient.java | 249 +- .../client/params/AbstractActionArgs.java | 34 +- .../client/params/AbstractArgsDelegate.java | 28 + .../AbstractClusterBuildingActionArgs.java | 58 + .../service/client/params/ActionBuildArgs.java |2 - .../service/client/params/ActionClientArgs.java | 71 + .../service/client/params/ActionCreateArgs.java |2 - .../client/params/ActionDependencyArgs.java |4 +- .../service/client/params/ActionExistsArgs.java | 49 + .../service/client/params/ActionFlexArgs.java |7 +- .../service/client/params/ActionFreezeArgs.java | 56 + .../service/client/params/ActionHelpArgs.java | 44 + .../service/client/params/ActionKDiagArgs.java | 76 + .../service/client/params/ActionKeytabArgs.java | 76 + .../service/client/params/ActionListArgs.java | 76 + .../client/params/ActionRegistryArgs.java | 218 ++ .../client/params/ActionResolveArgs.java| 153 + .../client/params/ActionResourceArgs.java | 70 + .../service/client/params/ActionStatusArgs.java | 51 + .../service/client/params/ActionThawArgs.java | 67 + .../service/client/params/ActionTokensArgs.java | 78 + .../service/client/params/ActionUpdateArgs.java | 32 + .../yarn/service/client/params/ArgOps.java |4 +- .../yarn/service/client/params/Arguments.java | 37 +- .../yarn/service/client/params/ClientArgs.java | 147 +- .../yarn/service/client/params/CommonArgs.java | 30 +- .../client/params/ComponentArgsDelegate.java|4 +- .../client/params/DontSplitArguments.java | 34 + .../client/params/LaunchArgsAccessor.java | 30 + .../client/params/LaunchArgsDelegate.java | 51 + .../client/params/OptionArgsDelegate.java | 66 + .../client/params/PathArgumentConverter.java| 34 + .../client/params/SliderAMCreateAction.java |4 - .../service/client/params/SliderActions.java| 48 +-
[05/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java deleted file mode 100644 index ece65ba..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history; - -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockFactory; -import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; -import org.apache.slider.server.appmaster.state.ContainerOutcome; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Testing finding nodes for new instances. - * - * This stresses the non-AA codepath - */ -public class TestRoleHistoryFindNodesForNewInstances extends -BaseMockAppStateTest { - private static final Logger LOG = - LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class); - - public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException { - } - - @Override - public String getTestName() { -return "TestFindNodesForNewInstances"; - } - - private NodeInstance age1Active4; - private NodeInstance age2Active2; - private NodeInstance age3Active0; - private NodeInstance age4Active1; - private NodeInstance age2Active0; - - private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); - - private RoleStatus roleStat; - private RoleStatus roleStat2; - - @Override - public void setup() throws Exception { -super.setup(); - -age1Active4 = nodeInstance(1, 4, 0, 0); -age2Active2 = nodeInstance(2, 2, 0, 1); -age3Active0 = nodeInstance(3, 0, 0, 0); -age4Active1 = nodeInstance(4, 1, 0, 0); -age2Active0 = nodeInstance(2, 0, 0, 0); - -roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1, -age1Active4, age3Active0)); -roleHistory.buildRecentNodeLists(); - -roleStat = getRole0Status(); -roleStat2 = getRole2Status(); - } - - public List findNodes(int count) { -return findNodes(count, roleStat); - } - - public List findNodes(int count, RoleStatus roleStatus) { -List found = new ArrayList<>(); -for (int i = 0; i < count; i++) { - NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus); - if (f != null) { -found.add(f); - } -} -return found; - } - - //@Test - public void testFind1NodeR0() throws Throwable { -NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); -LOG.info("found: {}", found); -assertTrue(Arrays.asList(age3Active0).contains(found)); - } - - //@Test - public void testFind2NodeR0() throws Throwable { -NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); -LOG.info("found: {}", found); -assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found)); -NodeInstance found2 =
[61/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java new file mode 100644 index 000..add2475 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.registry; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; + +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join; + +/** + * Registry view for providers. This tracks where the service + * is registered, offers access to the record and other things. + */ +public class YarnRegistryViewForProviders { + private static final Log LOG = + LogFactory.getLog(YarnRegistryViewForProviders.class); + + private final RegistryOperations registryOperations; + private final String user; + private final String sliderServiceClass; + private final String instanceName; + /** + * Record used where the service registered itself. + * Null until the service is registered + */ + private ServiceRecord selfRegistration; + + /** + * Path where record was registered + * Null until the service is registered + */ + private String selfRegistrationPath; + + public YarnRegistryViewForProviders(RegistryOperations registryOperations, + String user, + String sliderServiceClass, + String instanceName, + ApplicationAttemptId applicationAttemptId) { +Preconditions.checkArgument(registryOperations != null, +"null registry operations"); +Preconditions.checkArgument(user != null, "null user"); +Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass), +"unset service class"); +Preconditions.checkArgument(SliderUtils.isSet(instanceName), +"instanceName"); +Preconditions.checkArgument(applicationAttemptId != null, +"null applicationAttemptId"); +this.registryOperations = registryOperations; +this.user = user; +this.sliderServiceClass = sliderServiceClass; +this.instanceName = instanceName; + } + + public String getUser() { +return user; + } + + + private void setSelfRegistration(ServiceRecord selfRegistration) { +this.selfRegistration = selfRegistration; + } + + /** + * Get the path to where the service has registered itself. + * Null until the service is registered + * @return the service registration path. + */ + public String getSelfRegistrationPath() { +return selfRegistrationPath; + } + + /** + * Get the absolute path to where the service has registered itself. + * This includes the base registry path + * Null until the service is registered + * @return the service
[30/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java deleted file mode 100644 index f1bf2ad..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.client; - -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; -import org.apache.slider.common.params.ActionAMSuicideArgs; -import org.apache.slider.common.params.ActionClientArgs; -import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; -import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; -import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionKeytabArgs; -import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.params.ActionKillContainerArgs; -import org.apache.slider.common.params.ActionListArgs; -import org.apache.slider.common.params.ActionRegistryArgs; -import org.apache.slider.common.params.ActionResolveArgs; -import org.apache.slider.common.params.ActionResourceArgs; -import org.apache.slider.common.params.ActionStatusArgs; -import org.apache.slider.common.params.ActionThawArgs; -import org.apache.slider.common.params.ActionUpgradeArgs; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; - -import java.io.IOException; - -/** - * Interface of those method calls in the slider API that are intended - * for direct public invocation. - * - * Stability: evolving - */ -public interface SliderClientAPI extends Service { - - int actionDestroy(String clustername) throws YarnException, IOException; - - /** - * AM to commit an asynchronous suicide - */ - int actionAmSuicide(String clustername, - ActionAMSuicideArgs args) throws YarnException, IOException; - - /** - * Manage keytabs leveraged by slider - * - * @param keytabInfo the arguments needed to manage the keytab - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionKeytab(ActionKeytabArgs keytabInfo) - throws YarnException, IOException; - - /** - * Manage file resources leveraged by slider - * - * @param resourceInfo the arguments needed to manage the resource - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionResource(ActionResourceArgs resourceInfo) - throws YarnException, IOException; - - /** - * Perform client operations such as install or configure - * - * @param clientInfo the arguments needed for client operations - * - * @throws SliderException bad arguments. - * @throws IOException problems related to package and destination folders - */ - int actionClient(ActionClientArgs clientInfo) - throws IOException, YarnException; - - /** - * Update the cluster specification - * - * @param clustername cluster name - * @param buildInfo the arguments needed to update the cluster - * @throws YarnException Yarn problems - * @throws
[02/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java deleted file mode 100644 index a667432..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.services.workflow; - -import org.junit.Test; - - -public class TestWorkflowServiceTerminatingRunnable extends WorkflowServiceTestBase { - - //@Test - public void testNoservice() throws Throwable { - -try { - new ServiceTerminatingRunnable(null, new SimpleRunnable()); - fail("unexpected "); -} catch (IllegalArgumentException e) { - - // expected -} - } - - - //@Test - public void testBasicRun() throws Throwable { - -WorkflowCompositeService svc = run(new WorkflowCompositeService()); -ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc, -new SimpleRunnable()); - -// synchronous in-thread execution -runnable.run(); -assertStopped(svc); - } - - //@Test - public void testFailureRun() throws Throwable { - -WorkflowCompositeService svc = run(new WorkflowCompositeService()); -ServiceTerminatingRunnable runnable = -new ServiceTerminatingRunnable(svc, new SimpleRunnable(true)); - -// synchronous in-thread execution -runnable.run(); -assertStopped(svc); -assertNotNull(runnable.getException()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java deleted file mode 100644 index f38bd9d..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.services.workflow; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.Service; -import org.junit.Assert;
hadoop git commit: HADOOP-14670. Increase minimum cmake version for all platforms
Repository: hadoop Updated Branches: refs/heads/trunk ce79f7be2 -> 71bbb86d6 HADOOP-14670. Increase minimum cmake version for all platforms Signed-off-by: Chris DouglasProject: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71bbb86d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71bbb86d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71bbb86d Branch: refs/heads/trunk Commit: 71bbb86d69ac474596f5619d22718e9f7ff5f9dc Parents: ce79f7b Author: Allen Wittenauer Authored: Wed Jul 19 17:51:14 2017 -0700 Committer: Allen Wittenauer Committed: Wed Aug 30 21:47:24 2017 -0700 -- BUILDING.txt| 4 +-- dev-support/docker/Dockerfile | 17 ++--- .../hadoop-common/HadoopCommon.cmake| 26 .../hadoop-common/src/CMakeLists.txt| 2 +- .../bzip2/org_apache_hadoop_io_compress_bzip2.h | 2 ++ .../src/CMakeLists.txt | 2 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 2 ++ .../main/native/libhdfs-tests/CMakeLists.txt| 2 ++ .../src/main/native/libhdfs/CMakeLists.txt | 3 +++ .../src/CMakeLists.txt | 2 +- .../maven/plugin/cmakebuilder/CompileMojo.java | 2 -- hadoop-tools/hadoop-pipes/src/CMakeLists.txt| 2 +- .../src/CMakeLists.txt | 22 ++--- start-build-env.sh | 2 ++ 14 files changed, 49 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/71bbb86d/BUILDING.txt -- diff --git a/BUILDING.txt b/BUILDING.txt index f9cc842..14deec8 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ Requirements: * JDK 1.8+ * Maven 3.3 or later * ProtocolBuffer 2.5.0 -* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac +* CMake 3.1 or newer (if compiling native code) * Zlib devel (if compiling native code) * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance) * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs) @@ -345,7 +345,7 @@ Requirements: * JDK 1.8+ * Maven 3.0 or later * ProtocolBuffer 2.5.0 -* CMake 2.6 or newer +* CMake 3.1 or newer * Windows SDK 7.1 or Visual Studio 2010 Professional * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib) http://git-wip-us.apache.org/repos/asf/hadoop/blob/71bbb86d/dev-support/docker/Dockerfile -- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1775323..31ac611 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,7 +35,6 @@ ENV DEBCONF_TERSE true RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ build-essential \ bzip2 \ -cmake \ curl \ doxygen \ fuse \ @@ -89,11 +88,22 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant ## RUN mkdir -p /opt/maven && \ curl -L -s -S \ - http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ + https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ -o /opt/maven.tar.gz && \ tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven ENV MAVEN_HOME /opt/maven -ENV PATH "$PATH:/opt/maven/bin" +ENV PATH "${PATH}:/opt/maven/bin" + +## +# Install cmake +## +RUN mkdir -p /opt/cmake && \ +curl -L -s -S \ + https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \ + -o /opt/cmake.tar.gz && \ +tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake +ENV CMAKE_HOME /opt/cmake +ENV PATH "${PATH}:/opt/cmake/bin" ## # Install findbugs @@ -104,6 +114,7 @@ RUN mkdir -p /opt/findbugs && \ -o /opt/findbugs.tar.gz && \ tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs ENV FINDBUGS_HOME /opt/findbugs +ENV PATH "${PATH}:/opt/findbugs/bin" # Install shellcheck http://git-wip-us.apache.org/repos/asf/hadoop/blob/71bbb86d/hadoop-common-project/hadoop-common/HadoopCommon.cmake -- diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake index c46d9e5..faabeed 100644 --- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake +++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake @@ -117,19 +117,25 @@
hadoop git commit: YARN-7094. Document the current known issue with server-side NM graceful decom (rkanter)
Repository: hadoop Updated Branches: refs/heads/branch-2 9a0aac417 -> 8523319e1 YARN-7094. Document the current known issue with server-side NM graceful decom (rkanter) (cherry picked from commit ce79f7be29dfbad444614cb340c59e4ff8137e89) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8523319e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8523319e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8523319e Branch: refs/heads/branch-2 Commit: 8523319e15ce74c140fab4f827b110b2d32f8e37 Parents: 9a0aac4 Author: Robert KanterAuthored: Wed Aug 30 18:04:55 2017 -0700 Committer: Robert Kanter Committed: Wed Aug 30 18:06:03 2017 -0700 -- .../main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 3 ++- .../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8523319e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index ac5f559..24c1da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -113,7 +113,8 @@ public class RMAdminCLI extends HAAdmin { + " be handled by the client or the ResourceManager. The client" + "-side tracking is blocking, while the server-side tracking" + " is not. Omitting the timeout, or a timeout of -1, indicates" - + " an infinite timeout.")) + + " an infinite timeout. Known Issue: the server-side tracking" + + " will immediately decommission if an RM HA failover occurs.")) .put("-refreshNodesResources", new UsageInfo("", "Refresh resources of NodeManagers at the ResourceManager.")) .put("-refreshSuperUserGroupsConfiguration", new UsageInfo("", http://git-wip-us.apache.org/repos/asf/hadoop/blob/8523319e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 103d05c..1b36619 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -190,7 +190,7 @@ Usage: ``` Usage: yarn rmadmin -refreshQueues - -refreshNodes [-g [timeout in seconds]] + -refreshNodes [-g|graceful [timeout in seconds] -client|server] -refreshNodesResources -refreshSuperUserGroupsConfiguration -refreshUserToGroupsMappings @@ -215,7 +215,7 @@ Usage: | COMMAND\_OPTIONS | Description | |: |: | | -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. | -| -refreshNodes [-g|graceful [timeout in seconds] -client|server] | Refresh the hosts information at the ResourceManager. -g option indicates graceful decommission of excluded hosts, in which case, the optional timeout indicates maximal time in seconds ResourceManager should wait before forcefully mark the node as decommissioned. | +| -refreshNodes [-ggraceful [timeout in seconds] -clientserver] | Refresh the hosts information at the ResourceManager. Here [-ggraceful [timeout in seconds] -clientserver] is optional, if we specify the timeout then ResourceManager will wait for timeout before marking the NodeManager as decommissioned. The -clientserver indicates if the timeout tracking should be handled by the client or the ResourceManager. The client-side tracking is blocking, while the server-side tracking is not. Omitting the timeout, or a timeout of -1, indicates an infinite timeout. Known Issue: the server-side tracking will immediately decommission if an RM HA failover occurs. | | -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. | | -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. | | -refreshUserToGroupsMappings | Refresh user-to-groups mappings.
hadoop git commit: YARN-7094. Document the current known issue with server-side NM graceful decom (rkanter)
Repository: hadoop Updated Branches: refs/heads/trunk cc23514ab -> ce79f7be2 YARN-7094. Document the current known issue with server-side NM graceful decom (rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce79f7be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce79f7be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce79f7be Branch: refs/heads/trunk Commit: ce79f7be29dfbad444614cb340c59e4ff8137e89 Parents: cc23514 Author: Robert KanterAuthored: Wed Aug 30 18:04:55 2017 -0700 Committer: Robert Kanter Committed: Wed Aug 30 18:04:55 2017 -0700 -- .../main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 3 ++- .../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce79f7be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index ac5f559..24c1da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -113,7 +113,8 @@ public class RMAdminCLI extends HAAdmin { + " be handled by the client or the ResourceManager. The client" + "-side tracking is blocking, while the server-side tracking" + " is not. Omitting the timeout, or a timeout of -1, indicates" - + " an infinite timeout.")) + + " an infinite timeout. Known Issue: the server-side tracking" + + " will immediately decommission if an RM HA failover occurs.")) .put("-refreshNodesResources", new UsageInfo("", "Refresh resources of NodeManagers at the ResourceManager.")) .put("-refreshSuperUserGroupsConfiguration", new UsageInfo("", http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce79f7be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 478377fa..5f430ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -189,7 +189,7 @@ Usage: ``` Usage: yarn rmadmin -refreshQueues - -refreshNodes [-g [timeout in seconds]] + -refreshNodes [-g|graceful [timeout in seconds] -client|server] -refreshNodesResources -refreshSuperUserGroupsConfiguration -refreshUserToGroupsMappings @@ -214,7 +214,7 @@ Usage: | COMMAND\_OPTIONS | Description | |: |: | | -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. | -| -refreshNodes [-g|graceful [timeout in seconds] -client|server] | Refresh the hosts information at the ResourceManager. -g option indicates graceful decommission of excluded hosts, in which case, the optional timeout indicates maximal time in seconds ResourceManager should wait before forcefully mark the node as decommissioned. | +| -refreshNodes [-ggraceful [timeout in seconds] -clientserver] | Refresh the hosts information at the ResourceManager. Here [-ggraceful [timeout in seconds] -clientserver] is optional, if we specify the timeout then ResourceManager will wait for timeout before marking the NodeManager as decommissioned. The -clientserver indicates if the timeout tracking should be handled by the client or the ResourceManager. The client-side tracking is blocking, while the server-side tracking is not. Omitting the timeout, or a timeout of -1, indicates an infinite timeout. Known Issue: the server-side tracking will immediately decommission if an RM HA failover occurs. | | -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. | | -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. | | -refreshUserToGroupsMappings | Refresh user-to-groups mappings. | -
hadoop git commit: YARN-7115. Move BoundedAppender to org.hadoop.yarn.util pacakge (Contributed by Jian He via Daniel Templeton)
Repository: hadoop Updated Branches: refs/heads/trunk 3e0e2033c -> cc23514ab YARN-7115. Move BoundedAppender to org.hadoop.yarn.util pacakge (Contributed by Jian He via Daniel Templeton) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc23514a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc23514a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc23514a Branch: refs/heads/trunk Commit: cc23514abacb4d6589e731cc5ce5d8e6f19c955d Parents: 3e0e203 Author: Daniel TempletonAuthored: Wed Aug 30 17:26:13 2017 -0700 Committer: Daniel Templeton Committed: Wed Aug 30 17:26:13 2017 -0700 -- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +- .../hadoop/yarn/util/BoundedAppender.java | 142 +++ .../hadoop/yarn/util/TestBoundedAppender.java | 115 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 114 +-- .../rmapp/attempt/TestBoundedAppender.java | 116 --- .../TestRMAppAttemptImplDiagnostics.java| 3 +- 6 files changed, 270 insertions(+), 234 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc23514a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index f17cf8c..c2a5c67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -108,6 +108,15 @@ org.apache.hadoop hadoop-annotations + + + junit + junit + test + org.mockito mockito-all @@ -129,11 +138,6 @@ protobuf-java - junit - junit - test - - org.bouncycastle bcprov-jdk16 test http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc23514a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java new file mode 100644 index 000..917d696 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A {@link CharSequence} appender that considers its {@link #limit} as upper + * bound. + * + * When {@link #limit} would be reached on append, past messages will be + * truncated from head, and a header telling the user about truncation will be + * prepended, with ellipses in between header and messages. + * + * Note that header and ellipses are not counted against {@link #limit}. + * + * An example: + * + * + * {@code + * // At the beginning it's an empty string + * final Appendable shortAppender = new BoundedAppender(80); + * // The whole message fits into limit + * shortAppender.append( + * "message1 this is a very long message but fitting into limit\n"); + * // The first message is truncated, the second not + * shortAppender.append("message2 this is shorter than the previous one\n"); + * // The first message is deleted, the second truncated, the third + * // preserved + * shortAppender.append("message3 this is even shorter message, maybe.\n"); + * // The first two are deleted, the third one truncated, the
hadoop git commit: HDFS-9107. Prevent NN's unrecoverable death spiral after full GC (Daryn Sharp via Colin P. McCabe)
Repository: hadoop Updated Branches: refs/heads/branch-2.7 70df729a1 -> c2350ec42 HDFS-9107. Prevent NN's unrecoverable death spiral after full GC (Daryn Sharp via Colin P. McCabe) (cherry-picked from 4e7c6a653f108d44589f84d78a03d92ee0e8a3c3) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2350ec4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2350ec4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2350ec4 Branch: refs/heads/branch-2.7 Commit: c2350ec42d9389fc6374a395574f4bdd646209c0 Parents: 70df729 Author: Colin Patrick MccabeAuthored: Fri Sep 25 15:25:42 2015 -0700 Committer: Konstantin V Shvachko Committed: Wed Aug 30 16:15:03 2017 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../blockmanagement/HeartbeatManager.java | 30 ++-- .../blockmanagement/TestHeartbeatHandling.java | 27 ++ 3 files changed, 58 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2350ec4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 845a964..d1ebdcd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -19,6 +19,9 @@ Release 2.7.5 - UNRELEASED HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. (Vinayakumar B. via kihwal) +HDFS-9107. Prevent NN's unrecoverable death spiral after full GC (Daryn +Sharp via Colin P. McCabe) + Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2350ec4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index dfc3c60..93e9b32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -28,10 +29,13 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Manage the heartbeats received from datanodes. * The datanode list and statistics are synchronized @@ -56,8 +60,8 @@ class HeartbeatManager implements DatanodeStatistics { private final long heartbeatRecheckInterval; /** Heartbeat monitor thread */ private final Daemon heartbeatThread = new Daemon(new Monitor()); + private final StopWatch heartbeatStopWatch = new StopWatch(); - final Namesystem namesystem; final BlockManager blockManager; @@ -245,7 +249,18 @@ class HeartbeatManager implements DatanodeStatistics { stats.add(node); } } - + + @VisibleForTesting + void restartHeartbeatStopWatch() { +heartbeatStopWatch.reset().start(); + } + + @VisibleForTesting + boolean shouldAbortHeartbeatCheck(long offset) { +long elapsed = heartbeatStopWatch.now(TimeUnit.MILLISECONDS); +return elapsed + offset > heartbeatRecheckInterval; + } + /** * Check if there are any expired heartbeats, and if so, * whether any blocks have to be re-replicated. @@ -292,6 +307,10 @@ class HeartbeatManager implements DatanodeStatistics { int numOfStaleStorages = 0; synchronized(this) { for (DatanodeDescriptor d : datanodes) { + // check if an excessive GC pause has occurred + if (shouldAbortHeartbeatCheck(0)) { +return; + } if (dead == null && dm.isDatanodeDead(d)) { stats.incrExpiredHeartbeats(); dead = d; @@ -360,6 +379,7 @@ class HeartbeatManager implements DatanodeStatistics { @Override public void run() {
[32/50] [abbrv] hadoop git commit: YARN-6935. [YARN-3926] ResourceProfilesManagerImpl.parseResource() has no need of the key parameter (Contributed by Manikandan R via Daniel Templeton)
YARN-6935. [YARN-3926] ResourceProfilesManagerImpl.parseResource() has no need of the key parameter (Contributed by Manikandan R via Daniel Templeton) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ae857b3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ae857b3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ae857b3 Branch: refs/heads/YARN-3926 Commit: 2ae857b38fcd93f395f020b112cbc241701b85eb Parents: 19a09a0 Author: Daniel TempletonAuthored: Fri Aug 11 16:32:13 2017 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../resource/ResourceProfilesManagerImpl.java | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ae857b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index ab6..b5ab384 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -87,22 +87,22 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { Iterator iterator = data.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = (Map.Entry) iterator.next(); - String key = entry.getKey().toString(); - if (key.isEmpty()) { + String profileName = entry.getKey().toString(); + if (profileName.isEmpty()) { throw new IOException( "Name of resource profile cannot be an empty string"); } if (entry.getValue() instanceof Map) { -Map value = (Map) entry.getValue(); +Map profileInfo = (Map) entry.getValue(); // ensure memory and vcores are specified -if (!value.containsKey(MEMORY) || !value.containsKey(VCORES)) { +if (!profileInfo.containsKey(MEMORY) || !profileInfo.containsKey(VCORES)) { throw new IOException( - "Illegal resource profile definition; profile '" + key + "Illegal resource profile definition; profile '" + profileName + "' must contain '" + MEMORY + "' and '" + VCORES + "'"); } -Resource resource = parseResource(key, value); -profiles.put(key, resource); -LOG.info("Added profile '" + key + "' with resources " + resource); +Resource resource = parseResource(profileInfo); +profiles.put(profileName, resource); +LOG.info("Added profile '" + profileName + "' with resources " + resource); } } // check to make sure mandatory profiles are present @@ -116,9 +116,9 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { LOG.info("Loaded profiles " + profiles.keySet()); } - private Resource parseResource(String key, Map value) throws IOException { + private Resource parseResource(Map profileInfo) throws IOException { Resource resource = Resource.newInstance(0, 0); -Iterator iterator = value.entrySet().iterator(); +Iterator iterator = profileInfo.entrySet().iterator(); Map resourceTypes = ResourceUtils .getResourceTypes(); while (iterator.hasNext()) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-7039. Fix javac and javadoc errors in YARN-3926 branch. (Sunil G via wangda)
YARN-7039. Fix javac and javadoc errors in YARN-3926 branch. (Sunil G via wangda) Change-Id: I442bf6d838b3aba83f1f6779cf9dcf8596a2102d Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/425250a7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/425250a7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/425250a7 Branch: refs/heads/YARN-3926 Commit: 425250a772a9f40dc7d03fdfe02a5022ccc821d7 Parents: 6123f60 Author: Wangda TanAuthored: Tue Aug 22 16:18:01 2017 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:51:18 2017 -0700 -- .../src/main/resources/META-INF/LICENSE.txt | 1661 -- .../src/main/resources/META-INF/NOTICE.txt | 283 --- .../records/ApplicationResourceUsageReport.java |1 - .../yarn/api/records/ResourceTypeInfo.java |6 +- .../yarn/api/records/impl/package-info.java |4 + .../hadoop/yarn/util/UnitsConversionUtil.java |5 +- .../yarn/util/resource/ResourceUtils.java |4 +- .../hadoop/yarn/util/resource/package-info.java |4 + .../yarn/conf/TestResourceInformation.java |3 + .../yarn/util/TestUnitsConversionUtil.java |8 +- .../applications/distributedshell/Client.java | 20 +- .../yarn/client/api/impl/AMRMClientImpl.java|1 - .../GetAllResourceProfilesResponsePBImpl.java |5 - .../pb/GetAllResourceTypeInfoRequestPBImpl.java |3 +- .../pb/GetResourceProfileRequestPBImpl.java |5 - .../pb/GetResourceProfileResponsePBImpl.java|5 - .../ApplicationResourceUsageReportPBImpl.java |1 - .../impl/pb/ProfileCapabilityPBImpl.java|5 - .../records/impl/pb/ResourceTypeInfoPBImpl.java |2 - .../hadoop/yarn/util/resource/Resources.java|6 +- .../hadoop/yarn/api/TestPBImplRecords.java |4 - .../yarn/util/resource/TestResourceUtils.java | 47 +- .../nodemanager/NodeStatusUpdaterImpl.java |8 +- .../util/NodeManagerHardwareUtils.java | 36 +- .../util/TestNodeManagerHardwareUtils.java |2 +- .../ApplicationMasterService.java |1 - .../server/resourcemanager/ResourceManager.java |2 +- .../resource/ResourceProfilesManagerImpl.java |8 +- .../resource/TestResourceProfiles.java |8 +- 29 files changed, 100 insertions(+), 2048 deletions(-) -- - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev.
YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/071bcfb5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/071bcfb5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/071bcfb5 Branch: refs/heads/YARN-3926 Commit: 071bcfb56b3ba06e86ac5c8100d93bcb4a27fd9f Parents: 92666c7 Author: Varun VasudevAuthored: Sat Oct 8 19:43:33 2016 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 23 +++ .../yarn/conf/TestYarnConfigurationFields.java | 4 + .../src/main/resources/yarn-default.xml | 16 ++ .../hadoop-yarn-server-resourcemanager/pom.xml | 5 + .../resource/ResourceProfilesManager.java | 46 + .../resource/ResourceProfilesManagerImpl.java | 176 +++ .../resource/TestResourceProfiles.java | 142 +++ .../resources/profiles/illegal-profiles-1.json | 10 ++ .../resources/profiles/illegal-profiles-2.json | 10 ++ .../resources/profiles/illegal-profiles-3.json | 10 ++ .../resources/profiles/sample-profiles-1.json | 14 ++ .../resources/profiles/sample-profiles-2.json | 26 +++ 12 files changed, 482 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/071bcfb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1fc075e..c8c61ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -879,6 +879,29 @@ public class YarnConfiguration extends Configuration { public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; /** + * Enable/disable resource profiles. + */ + @Public + @Unstable + public static final String RM_RESOURCE_PROFILES_ENABLED = + RM_PREFIX + "resource-profiles.enabled"; + @Public + @Unstable + public static final boolean DEFAULT_RM_RESOURCE_PROFILES_ENABLED = false; + + /** + * File containing resource profiles. + */ + @Public + @Unstable + public static final String RM_RESOURCE_PROFILES_SOURCE_FILE = + RM_PREFIX + "resource-profiles.source-file"; + @Public + @Unstable + public static final String DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE = + "resource-profiles.json"; + + /** * Timeout in seconds for YARN node graceful decommission. * This is the maximal time to wait for running containers and applications * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED. http://git-wip-us.apache.org/repos/asf/hadoop/blob/071bcfb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index bd7bf93..ce1239d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -141,6 +141,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { // Used as Java command line properties, not XML configurationPrefixToSkipCompare.add("yarn.app.container"); +// Ignore default file name for resource profiles +configurationPropsToSkipCompare +.add(YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE); + // Ignore NodeManager "work in progress" variables configurationPrefixToSkipCompare .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED); http://git-wip-us.apache.org/repos/asf/hadoop/blob/071bcfb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
[21/50] [abbrv] hadoop git commit: YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)
YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19378e4d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19378e4d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19378e4d Branch: refs/heads/YARN-3926 Commit: 19378e4db67d0e146d9a4e0dbd33a35d1256b75d Parents: f28b755 Author: Arun SureshAuthored: Thu Mar 17 23:50:22 2016 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../yarn/api/records/ResourceInformation.java | 2 +- .../hadoop/yarn/util/UnitsConversionUtil.java | 45 ++-- .../yarn/util/TestUnitsConversionUtil.java | 17 +++- .../api/records/impl/pb/ResourcePBImpl.java | 2 +- .../yarn/util/resource/TestResourceUtils.java | 2 +- 5 files changed, 52 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19378e4d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 80e3192..a17e81b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable { private static final String VCORES_URI = "vcores"; public static final ResourceInformation MEMORY_MB = - ResourceInformation.newInstance(MEMORY_URI, "M"); + ResourceInformation.newInstance(MEMORY_URI, "Mi"); public static final ResourceInformation VCORES = ResourceInformation.newInstance(VCORES_URI); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19378e4d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 7785263..47bb3df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -46,7 +46,8 @@ public class UnitsConversionUtil { } private static final String[] UNITS = - {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"}; + { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti", + "Pi" }; private static final List SORTED_UNITS = Arrays.asList(UNITS); public static final Set KNOWN_UNITS = createKnownUnitsSet(); private static final Converter PICO = @@ -65,6 +66,15 @@ public class UnitsConversionUtil { private static final Converter PETA = new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L); + private static final Converter KILO_BINARY = new Converter(1024L, 1L); + private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L); + private static final Converter GIGA_BINARY = + new Converter(1024L * 1024L * 1024L, 1L); + private static final Converter TERA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L, 1L); + private static final Converter PETA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L); + private static Set createKnownUnitsSet() { Set ret = new HashSet<>(); ret.addAll(Arrays.asList(UNITS)); @@ -93,6 +103,16 @@ public class UnitsConversionUtil { return TERA; case "P": return PETA; +case "Ki": + return KILO_BINARY; +case "Mi": + return MEGA_BINARY; +case "Gi": + return GIGA_BINARY; +case "Ti": + return TERA_BINARY; +case "Pi": + return PETA_BINARY; default: throw new IllegalArgumentException( "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS); @@ -112,28 +132,29 @@ public class UnitsConversionUtil { if (toUnit == null || fromUnit == null || fromValue == null) { throw new IllegalArgumentException("One or more arguments are null"); } -Long tmp; String overflowMsg =
[42/50] [abbrv] hadoop git commit: YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev.
YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61b784c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61b784c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61b784c4 Branch: refs/heads/YARN-3926 Commit: 61b784c40042be1d68df0b9f622dca422a5153d3 Parents: aae6515 Author: Sunil GAuthored: Tue Apr 25 11:53:11 2017 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../hadoop/yarn/api/records/Resource.java | 19 +-- .../yarn/api/records/ResourceInformation.java | 51 ++--- .../hadoop/yarn/util/UnitsConversionUtil.java | 34 ++- .../yarn/conf/TestResourceInformation.java | 4 +- .../yarn/util/TestUnitsConversionUtil.java | 60 ++-- .../api/records/impl/pb/ResourcePBImpl.java | 23 .../resource/DominantResourceCalculator.java| 54 -- .../hadoop/yarn/util/resource/Resources.java| 18 +++--- .../yarn/util/resource/TestResourceUtils.java | 1 + .../yarn/util/resource/TestResources.java | 12 +++- .../resource/ResourceProfilesManagerImpl.java | 3 +- 11 files changed, 157 insertions(+), 122 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/61b784c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index c349a32..4356986 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -107,12 +107,23 @@ public abstract class Resource implements Comparable { @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { Resource ret = Resource.newInstance(0, 0); -for (Map.Entry entry : resource.getResources() +Resource.copy(resource, ret); +return ret; + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static void copy(Resource source, Resource dest) { +for (Map.Entry entry : source.getResources() .entrySet()) { - ret.setResourceInformation(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); + try { +ResourceInformation.copy(entry.getValue(), +dest.getResourceInformation(entry.getKey())); + } catch (YarnException ye) { +dest.setResourceInformation(entry.getKey(), +ResourceInformation.newInstance(entry.getValue())); + } } -return ret; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/61b784c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 7d74efc..d75b441 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -30,9 +30,9 @@ public class ResourceInformation implements Comparable { private String name; private String units; private ResourceTypes resourceType; - private Long value; - private Long minimumAllocation; - private Long maximumAllocation; + private long value; + private long minimumAllocation; + private long maximumAllocation; private static final String MEMORY_URI = "memory-mb"; private static final String VCORES_URI = "vcores"; @@ -106,7 +106,7 @@ public class ResourceInformation implements Comparable { * * @return the resource value */ - public Long getValue() { + public long getValue() { return value; } @@ -115,7 +115,7 @@ public class ResourceInformation implements Comparable { * * @param rValue the resource value */ - public void setValue(Long rValue) { + public void setValue(long rValue) {
[22/50] [abbrv] hadoop git commit: YARN-4715. Add support to read resource types from a config file. Contributed by Varun Vasudev.
YARN-4715. Add support to read resource types from a config file. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f28b755e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f28b755e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f28b755e Branch: refs/heads/YARN-3926 Commit: f28b755e2214c254c514df7c51110fa48763071f Parents: 450a5bf Author: Varun VasudevAuthored: Fri Mar 11 15:03:15 2016 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../hadoop/yarn/api/records/Resource.java | 24 +- .../yarn/api/records/ResourceInformation.java | 8 +- .../hadoop/yarn/conf/YarnConfiguration.java | 12 + .../exceptions/ResourceNotFoundException.java | 2 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 + .../api/records/impl/pb/ResourcePBImpl.java | 72 ++ .../resource/DominantResourceCalculator.java| 5 +- .../yarn/util/resource/ResourceUtils.java | 229 + .../hadoop/yarn/util/resource/Resources.java| 18 +- .../src/main/resources/yarn-default.xml | 10 + .../yarn/util/resource/TestResourceUtils.java | 248 +++ .../resource-types/resource-types-1.xml | 18 ++ .../resource-types/resource-types-2.xml | 29 +++ .../resource-types/resource-types-3.xml | 24 ++ .../resource-types/resource-types-4.xml | 34 +++ .../resource-types/resource-types-error-1.xml | 29 +++ .../resource-types/resource-types-error-2.xml | 29 +++ .../resource-types/resource-types-error-3.xml | 29 +++ .../resource-types/resource-types-error-4.xml | 24 ++ 19 files changed, 762 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f28b755e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 2371b13..ee8ef03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -101,15 +101,6 @@ public abstract class Resource implements Comparable { return new SimpleResource(memory, vCores); } - @Public - @Stable - public static Resource newInstance( - Map resources) { -Resource resource = Records.newRecord(Resource.class); -resource.setResources(resources); -return resource; - } - /** * This method is DEPRECATED: * Use {@link Resource#getMemorySize()} instead @@ -234,15 +225,6 @@ public abstract class Resource implements Comparable { public abstract Long getResourceValue(String resource) throws YarnException; /** - * Set the resources to the map specified. - * - * @param resources Desired resources - */ - @Public - @Evolving - public abstract void setResources(Map resources); - - /** * Set the ResourceInformation object for a particular resource. * * @param resource the resource for which the ResourceInformation is provided @@ -276,8 +258,8 @@ public abstract class Resource implements Comparable { result = prime * result + getVirtualCores(); for (Map.Entry entry : getResources() .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) || entry - .getKey().equals(ResourceInformation.VCORES.getName())) { + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getKey().equals(ResourceInformation.VCORES.getName())) { continue; } result = prime * result + entry.getValue().hashCode(); @@ -320,7 +302,7 @@ public abstract class Resource implements Comparable { .append(getVirtualCores()); for (Map.Entry entry : getResources() .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) && entry.getValue().getUnits() .equals(ResourceInformation.MEMORY_MB.getUnits())) { continue; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f28b755e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
[14/50] [abbrv] hadoop git commit: MAPREDUCE-6931. Remove TestDFSIO "Total Throughput" calculation. Contributed by Dennis Huo.
MAPREDUCE-6931. Remove TestDFSIO "Total Throughput" calculation. Contributed by Dennis Huo. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e0e2033 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e0e2033 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e0e2033 Branch: refs/heads/YARN-3926 Commit: 3e0e2033cc28d48a7909822416478aff7487bfe6 Parents: 4148023 Author: Konstantin V ShvachkoAuthored: Tue Aug 22 11:05:47 2017 -0700 Committer: Konstantin V Shvachko Committed: Wed Aug 30 14:07:23 2017 -0700 -- .../src/test/java/org/apache/hadoop/fs/TestDFSIO.java| 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e0e2033/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index 12fbdad..1d495c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -855,7 +855,7 @@ public class TestDFSIO implements Tool { long tStart = System.currentTimeMillis(); sequentialTest(fs, testType, nrBytes, nrFiles); long execTime = System.currentTimeMillis() - tStart; - String resultLine = "Seq Test exec time sec: " + (float)execTime / 1000; + String resultLine = "Seq Test exec time sec: " + msToSecs(execTime); LOG.info(resultLine); return 0; } @@ -919,6 +919,10 @@ public class TestDFSIO implements Tool { return ((float)bytes)/MEGA; } + static float msToSecs(long timeMillis) { +return timeMillis / 1000.0f; + } + private boolean checkErasureCodePolicy(String erasureCodePolicyName, FileSystem fs, TestType testType) throws IOException { Collection list = @@ -1051,11 +1055,10 @@ public class TestDFSIO implements Tool { "Date & time: " + new Date(System.currentTimeMillis()), "Number of files: " + tasks, " Total MBytes processed: " + df.format(toMB(size)), -" Throughput mb/sec: " + df.format(size * 1000.0 / (time * MEGA)), -"Total Throughput mb/sec: " + df.format(toMB(size) / ((float)execTime)), +" Throughput mb/sec: " + df.format(toMB(size) / msToSecs(time)), " Average IO rate mb/sec: " + df.format(med), " IO rate std deviation: " + df.format(stdDev), -" Test exec time sec: " + df.format((float)execTime / 1000), +" Test exec time sec: " + df.format(msToSecs(execTime)), "" }; PrintStream res = null; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-4172. Extend DominantResourceCalculator to account for all resources. (Varun Vasudev via wangda)
YARN-4172. Extend DominantResourceCalculator to account for all resources. (Varun Vasudev via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/450a5bf4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/450a5bf4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/450a5bf4 Branch: refs/heads/YARN-3926 Commit: 450a5bf412a5e3f8258abc85f88127b97b48fbbd Parents: 37148de Author: Wangda TanAuthored: Fri Jan 29 10:53:31 2016 +0800 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../resource/DominantResourceCalculator.java| 380 +-- 1 file changed, 273 insertions(+), 107 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/450a5bf4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 7697e1d..a94e7a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -22,25 +22,31 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.HashSet; +import java.util.Set; /** - * A {@link ResourceCalculator} which uses the concept of + * A {@link ResourceCalculator} which uses the concept of * dominant resource to compare multi-dimensional resources. * - * Essentially the idea is that the in a multi-resource environment, - * the resource allocation should be determined by the dominant share - * of an entity (user or queue), which is the maximum share that the - * entity has been allocated of any resource. - * - * In a nutshell, it seeks to maximize the minimum dominant share across - * all entities. - * + * Essentially the idea is that the in a multi-resource environment, + * the resource allocation should be determined by the dominant share + * of an entity (user or queue), which is the maximum share that the + * entity has been allocated of any resource. + * + * In a nutshell, it seeks to maximize the minimum dominant share across + * all entities. + * * For example, if user A runs CPU-heavy tasks and user B runs - * memory-heavy tasks, it attempts to equalize CPU share of user A - * with Memory-share of user B. - * + * memory-heavy tasks, it attempts to equalize CPU share of user A + * with Memory-share of user B. + * * In the single resource case, it reduces to max-min fairness for that resource. - * + * * See the Dominant Resource Fairness paper for more details: * www.cs.berkeley.edu/~matei/papers/2011/nsdi_drf.pdf */ @@ -50,6 +56,56 @@ public class DominantResourceCalculator extends ResourceCalculator { private static final Log LOG = LogFactory.getLog(DominantResourceCalculator.class); + + private Set resourceNames; + + public DominantResourceCalculator() { +resourceNames = new HashSet<>(); +resourceNames.add(ResourceInformation.MEMORY.getName()); +resourceNames.add(ResourceInformation.VCORES.getName()); + } + + /** + * Compare two resources - if the value for every resource type for the lhs + * is greater than that of the rhs, return 1. If the value for every resource + * type in the lhs is less than the rhs, return -1. Otherwise, return 0 + * + * @param lhs resource to be compared + * @param rhs resource to be compared + * @return 0, 1, or -1 + */ + private int compare(Resource lhs, Resource rhs) { +boolean lhsGreater = false; +boolean rhsGreater = false; +int ret = 0; + +for (String rName : resourceNames) { + try { +ResourceInformation lhsResourceInformation = +lhs.getResourceInformation(rName); +ResourceInformation rhsResourceInformation = +rhs.getResourceInformation(rName); +int diff = lhsResourceInformation.compareTo(rhsResourceInformation); +if (diff >= 1) { + lhsGreater = true; +}
[27/50] [abbrv] hadoop git commit: YARN-5587. Add support for resource profiles. (vvasudev via asuresh)
YARN-5587. Add support for resource profiles. (vvasudev via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/327a017e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/327a017e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/327a017e Branch: refs/heads/YARN-3926 Commit: 327a017ef12b18c5252cd44275ec51a86edf5846 Parents: 75a175a Author: Arun SureshAuthored: Tue Nov 15 01:01:07 2016 -0800 Committer: Wangda Tan Committed: Wed Aug 30 15:49:01 2017 -0700 -- .../dev-support/findbugs-exclude.xml| 4 + .../RegisterApplicationMasterResponse.java | 8 + .../yarn/api/records/ProfileCapability.java | 94 ++- .../hadoop/yarn/api/records/Resource.java | 14 ++ .../yarn/api/records/ResourceInformation.java | 57 ++- .../yarn/api/records/ResourceRequest.java | 43 - .../hadoop-yarn/hadoop-yarn-client/pom.xml | 1 + .../hadoop/yarn/client/api/AMRMClient.java | 117 +- .../yarn/client/api/impl/AMRMClientImpl.java| 152 ++--- .../client/api/impl/RemoteRequestsTable.java| 109 + .../yarn/client/api/impl/TestAMRMClient.java| 141 ++-- .../impl/TestAMRMClientContainerRequest.java| 8 +- .../api/impl/TestDistributedScheduling.java | 12 +- .../yarn/client/api/impl/TestNMClient.java | 5 +- .../TestOpportunisticContainerAllocation.java | 31 ++-- .../src/test/resources/resource-profiles.json | 18 +++ ...RegisterApplicationMasterResponsePBImpl.java | 58 +++ .../api/records/impl/pb/ResourcePBImpl.java | 4 +- .../records/impl/pb/ResourceRequestPBImpl.java | 41 - .../yarn/util/resource/ResourceUtils.java | 161 ++- .../hadoop/yarn/util/resource/Resources.java| 10 +- .../ApplicationMasterService.java | 1 + .../resourcemanager/DefaultAMSProcessor.java| 8 + .../server/resourcemanager/RMServerUtils.java | 50 ++ .../resource/ResourceProfilesManagerImpl.java | 4 + .../scheduler/AbstractYarnScheduler.java| 48 +- .../scheduler/ClusterNodeTracker.java | 3 +- .../scheduler/SchedulerUtils.java | 10 ++ .../scheduler/capacity/CapacityScheduler.java | 4 +- .../scheduler/fair/FairScheduler.java | 4 +- .../scheduler/fifo/FifoScheduler.java | 13 +- .../yarn/server/resourcemanager/MockRM.java | 2 + .../server/resourcemanager/TestAppManager.java | 1 + .../TestApplicationMasterService.java | 35 .../scheduler/fair/TestFairScheduler.java | 4 + .../hadoop/yarn/server/MiniYARNCluster.java | 2 + 36 files changed, 1102 insertions(+), 175 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/327a017e/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 6825a36..ce7a9c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -154,6 +154,10 @@ + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/327a017e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java index 0b886dd..8fa8563 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java @@ -204,4 +204,12 @@ public abstract class RegisterApplicationMasterResponse { @Unstable public abstract void setSchedulerResourceTypes( EnumSet types); + + @Public + @Unstable + public abstract Map getResourceProfiles(); + + @Private + @Unstable + public abstract void setResourceProfiles(Map profiles); }
[47/50] [abbrv] hadoop git commit: YARN-7093. Improve log message in ResourceUtils. (Sunil G via wangda)
YARN-7093. Improve log message in ResourceUtils. (Sunil G via wangda) Change-Id: I88928a747ee3eec17bc76fb71e9aaa632d091f0f Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d76440e3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d76440e3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d76440e3 Branch: refs/heads/YARN-3926 Commit: d76440e3686e5caa0ba39bac52c7f6821ee71ecc Parents: 425250a Author: Wangda TanAuthored: Thu Aug 24 10:44:28 2017 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:51:18 2017 -0700 -- .../java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d76440e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index ed16104..94c2e97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -353,7 +353,7 @@ public class ResourceUtils { initializedResources = true; } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile -+ "'. Falling back to memory and vcores as resources", fe); ++ "'. Falling back to memory and vcores as resources."); initializeResourcesMap(conf); initializedResources = true; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev.
YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aae65152 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aae65152 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aae65152 Branch: refs/heads/YARN-3926 Commit: aae651528f35ca05b86272053ff9cf84abc2852c Parents: 81e4c92 Author: Sunil GAuthored: Mon Mar 6 11:34:20 2017 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../records/ApplicationResourceUsageReport.java | 58 ++- .../src/main/proto/yarn_protos.proto| 7 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 35 +++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 16 +- .../ApplicationResourceUsageReportPBImpl.java | 151 --- .../yarn/api/records/impl/pb/ProtoUtils.java| 34 + .../apache/hadoop/yarn/util/StringHelper.java | 36 + .../hadoop/yarn/api/BasePBImplRecordsTest.java | 12 ++ .../hadoop/yarn/api/TestPBImplRecords.java | 4 + ...pplicationHistoryManagerOnTimelineStore.java | 18 ++- .../hadoop/yarn/server/utils/BuilderUtils.java | 16 +- .../server/resourcemanager/RMAppManager.java| 8 +- .../server/resourcemanager/RMServerUtils.java | 10 +- .../resourcemanager/recovery/RMStateStore.java | 7 +- .../records/ApplicationAttemptStateData.java| 89 +-- .../pb/ApplicationAttemptStateDataPBImpl.java | 50 ++ .../server/resourcemanager/rmapp/RMAppImpl.java | 43 +++--- .../resourcemanager/rmapp/RMAppMetrics.java | 41 +++-- .../attempt/AggregateAppResourceUsage.java | 34 ++--- .../rmapp/attempt/RMAppAttemptImpl.java | 32 ++-- .../rmapp/attempt/RMAppAttemptMetrics.java | 106 ++--- .../rmcontainer/RMContainerImpl.java| 16 +- .../scheduler/SchedulerApplicationAttempt.java | 47 +++--- .../resourcemanager/webapp/RMAppBlock.java | 14 +- .../resourcemanager/webapp/dao/AppInfo.java | 28 .../webapp/dao/ResourceInfo.java| 36 - .../webapp/dao/SchedulerInfo.java | 2 +- .../yarn_server_resourcemanager_recovery.proto | 2 + .../server/resourcemanager/TestAppManager.java | 7 +- .../TestContainerResourceUsage.java | 7 +- .../applicationsmanager/MockAsm.java| 9 +- .../metrics/TestSystemMetricsPublisher.java | 15 +- .../TestSystemMetricsPublisherForV2.java| 22 ++- .../recovery/RMStateStoreTestBase.java | 8 +- .../recovery/TestZKRMStateStore.java| 23 +-- .../resourcemanager/webapp/TestAppPage.java | 8 +- .../webapp/TestRMWebAppFairScheduler.java | 5 +- .../DefaultClientRequestInterceptor.java| 16 ++ .../clientrm/FederationClientInterceptor.java | 15 ++ .../router/clientrm/RouterClientRMService.java | 18 +++ .../PassThroughClientRequestInterceptor.java| 16 ++ 41 files changed, 868 insertions(+), 253 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae65152/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 3cf8f3d..f9c8975 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -24,6 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; +import java.util.HashMap; +import java.util.Map; + /** * Contains various scheduling metrics to be reported by UI and CLI. */ @@ -35,9 +38,9 @@ public abstract class ApplicationResourceUsageReport { @Unstable public static ApplicationResourceUsageReport newInstance( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc, - long preemptedMemorySeconds, long preemptedVcoresSeconds) { + Resource reservedResources,
[29/50] [abbrv] hadoop git commit: YARN-6786. [YARN-3926] ResourcePBImpl imports cleanup. Contributed by Yeliang Cang.
YARN-6786. [YARN-3926] ResourcePBImpl imports cleanup. Contributed by Yeliang Cang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5621b946 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5621b946 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5621b946 Branch: refs/heads/YARN-3926 Commit: 5621b94674ef34b73572c3df328f06aaf5b131bf Parents: 984fa07 Author: Sunil GAuthored: Thu Jul 13 16:30:59 2017 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5621b946/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index a9abed9..7bc7f5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.api.records.impl.pb; -import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -34,7 +33,10 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Collections; + @Private @Unstable - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-12356. Unit test for JournalNode sync during Rolling Upgrade. Contributed by Hanisha Koneru.
HDFS-12356. Unit test for JournalNode sync during Rolling Upgrade. Contributed by Hanisha Koneru. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd66a243 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd66a243 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd66a243 Branch: refs/heads/YARN-3926 Commit: fd66a243bfffc8260bfd69058625d4d9509cafe6 Parents: d04f85f Author: Arpit AgarwalAuthored: Wed Aug 30 10:29:42 2017 -0700 Committer: Arpit Agarwal Committed: Wed Aug 30 10:29:42 2017 -0700 -- .../qjournal/server/TestJournalNodeSync.java| 176 +++ 1 file changed, 137 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd66a243/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java index 2964f05..09ef3a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java @@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.qjournal.server; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; @@ -54,10 +56,11 @@ public class TestJournalNodeSync { private MiniQJMHACluster qjmhaCluster; private MiniDFSCluster dfsCluster; private MiniJournalCluster jCluster; - private FileSystem fs; private FSNamesystem namesystem; private int editsPerformed = 0; private final String jid = "ns1"; + private int activeNNindex=0; + private static final int DFS_HA_TAILEDITS_PERIOD_SECONDS=1; @Rule public TestName testName = new TestName(); @@ -71,13 +74,16 @@ public class TestJournalNodeSync { "testSyncAfterJNdowntimeWithoutQJournalQueue")) { conf.setInt(DFSConfigKeys.DFS_QJOURNAL_QUEUE_SIZE_LIMIT_KEY, 0); } +if (testName.getMethodName().equals("testSyncDuringRollingUpgrade")) { + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, + DFS_HA_TAILEDITS_PERIOD_SECONDS); +} qjmhaCluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(2) .build(); dfsCluster = qjmhaCluster.getDfsCluster(); jCluster = qjmhaCluster.getJournalCluster(); dfsCluster.transitionToActive(0); -fs = dfsCluster.getFileSystem(0); namesystem = dfsCluster.getNamesystem(0); } @@ -192,36 +198,7 @@ public class TestJournalNodeSync { // the journals. @Test(timeout=6) public void testRandomJournalMissingLogs() throws Exception { -Random randomJournal = new Random(); - -List journalCurrentDirs = Lists.newArrayList(); - -for (int i = 0; i < 3; i++) { - journalCurrentDirs.add(new StorageDirectory(jCluster.getJournalDir(i, - jid)).getCurrentDir()); -} - -int count = 0; -long lastStartTxId; -int journalIndex; -List missingLogs = Lists.newArrayList(); -while (count < 5) { - lastStartTxId = generateEditLog(); - - // Delete the last edit log segment from randomly selected journal node - journalIndex = randomJournal.nextInt(3); - missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex), - lastStartTxId)); - - // Delete the last edit log segment from two journals for some logs - if (count % 2 == 0) { -journalIndex = (journalIndex + 1) % 3; -missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex), -lastStartTxId)); - } - - count++; -} +List missingLogs = deleteEditLogsFromRandomJN(); GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 3); } @@ -277,7 +254,8 @@ public class TestJournalNodeSync { */ @Test (timeout=300_000) public void testSyncAfterJNdowntimeWithoutQJournalQueue() throws
[23/50] [abbrv] hadoop git commit: YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev.
YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8da23de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8da23de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8da23de Branch: refs/heads/YARN-3926 Commit: c8da23de66ff7b98e43949ef62580742751361a3 Parents: 19378e4 Author: Varun VasudevAuthored: Sat Jun 11 14:33:46 2016 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../hadoop/yarn/api/records/Resource.java | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 19 ++- .../FileSystemBasedConfigurationProvider.java | 3 +- .../hadoop/yarn/LocalConfigurationProvider.java | 3 +- .../api/records/impl/pb/ResourcePBImpl.java | 53 +++--- .../yarn/util/resource/ResourceUtils.java | 168 +++ .../yarn/util/resource/TestResourceUtils.java | 29 +++- .../resource-types/node-resources-1.xml | 29 .../resource-types/node-resources-2.xml | 39 + .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../util/NodeManagerHardwareUtils.java | 52 ++ .../resourcemanager/ResourceTrackerService.java | 9 +- 12 files changed, 342 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8da23de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index ee8ef03..c9c6a7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -308,7 +308,8 @@ public abstract class Resource implements Comparable { continue; } if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits().equals("")) { + && entry.getValue().getUnits() + .equals(ResourceInformation.VCORES.getUnits())) { continue; } sb.append(", ").append(entry.getKey()).append(": ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8da23de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bb399bc..1fc075e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -65,6 +65,10 @@ public class YarnConfiguration extends Configuration { "resource-types.xml"; @Private + public static final String NODE_RESOURCES_CONFIGURATION_FILE = + "node-resources.xml"; + + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( RESOURCE_TYPES_CONFIGURATION_FILE, @@ -74,6 +78,16 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); + @Private + public static final List NM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + NODE_RESOURCES_CONFIGURATION_FILE, + DR_CONFIGURATION_FILE, + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @@ -112,12 +126,15 @@ public class YarnConfiguration extends Configuration { public static final String YARN_PREFIX = "yarn."; / - // Scheduler resource types configs + // Resource types configs public static final String RESOURCE_TYPES = YarnConfiguration.YARN_PREFIX + "resource-types"; + public static final String NM_RESOURCES_PREFIX = + YarnConfiguration.NM_PREFIX + "resource-type."; + /** Delay before deleting resource to ease debugging of NM issues */
[31/50] [abbrv] hadoop git commit: YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton)
YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19a09a0b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19a09a0b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19a09a0b Branch: refs/heads/YARN-3926 Commit: 19a09a0bd474063a0fd5a05d9571ba8a5ec57245 Parents: 5621b94 Author: Daniel TempletonAuthored: Fri Aug 4 08:42:34 2017 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../dev-support/findbugs-exclude.xml| 18 + .../yarn/api/records/ProfileCapability.java | 8 +- .../hadoop/yarn/api/records/Resource.java | 234 .../yarn/api/records/ResourceInformation.java | 13 +- .../yarn/api/records/impl/BaseResource.java | 133 + .../yarn/api/records/impl/package-info.java | 22 + .../hadoop/yarn/util/UnitsConversionUtil.java | 8 +- .../yarn/util/resource/ResourceUtils.java | 534 +++ .../hadoop/yarn/util/resource/package-info.java | 22 + .../yarn/client/api/impl/TestAMRMClient.java| 8 +- .../yarn/api/records/impl/pb/ProtoUtils.java| 5 +- .../api/records/impl/pb/ResourcePBImpl.java | 110 ++-- .../resource/DominantResourceCalculator.java| 67 ++- .../yarn/util/resource/ResourceUtils.java | 488 - .../hadoop/yarn/util/resource/Resources.java| 194 --- .../yarn/util/resource/TestResourceUtils.java | 14 +- .../yarn/util/resource/TestResources.java | 7 +- .../resource/ResourceProfilesManagerImpl.java | 8 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 11 +- .../scheduler/SchedulerApplicationAttempt.java | 9 +- .../webapp/dao/SchedulerInfo.java | 3 +- .../server/resourcemanager/TestAppManager.java | 1 + 22 files changed, 1045 insertions(+), 872 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19a09a0b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index ce7a9c6..a5b4021 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -603,4 +603,22 @@ + + + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/19a09a0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java index 1a8d1c3..2cb4670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java @@ -162,10 +162,10 @@ public abstract class ProfileCapability { if (capability.getProfileCapabilityOverride() != null && !capability.getProfileCapabilityOverride().equals(none)) { - for (Map.Entry entry : capability - .getProfileCapabilityOverride().getResources().entrySet()) { -if (entry.getValue() != null && entry.getValue().getValue() >= 0) { - resource.setResourceInformation(entry.getKey(), entry.getValue()); + for (ResourceInformation entry : capability + .getProfileCapabilityOverride().getResources()) { +if (entry != null && entry.getValue() >= 0) { + resource.setResourceInformation(entry.getName(), entry); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19a09a0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 9a8e2ec..a485a57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++
[38/50] [abbrv] hadoop git commit: YARN-7042. Clean up unit tests after YARN-6610. (Daniel Templeton via wangda)
YARN-7042. Clean up unit tests after YARN-6610. (Daniel Templeton via wangda) Change-Id: I8e40f704b6fcdd5b14faa9548a27986501044fa1 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0c6219f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0c6219f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0c6219f Branch: refs/heads/YARN-3926 Commit: b0c6219f5b4982866869b8ecef3b5d79141e455d Parents: 7b6327e Author: Wangda TanAuthored: Thu Aug 17 11:18:08 2017 -0700 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../resource/DominantResourceCalculator.java| 2 +- .../util/resource/TestResourceCalculator.java | 95 ++-- 2 files changed, 49 insertions(+), 48 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0c6219f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 40b38b9..1e99bc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -126,7 +126,7 @@ public class DominantResourceCalculator extends ResourceCalculator { diff = max[0] - max[1]; } else if (clusterRes.length == 2) { // Special case to handle the common scenario of only CPU and memory -// so the we can optimize for performance +// so that we can optimize for performance diff = calculateSharesForMandatoryResources(clusterRes, lhs, rhs, lhsShares, rhsShares); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0c6219f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java index 19e7f8d..5b4155c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java @@ -24,7 +24,7 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -44,13 +44,18 @@ public class TestResourceCalculator { { new DominantResourceCalculator() } }); } - @BeforeClass - public static void setup() { + @Before + public void setupNoExtraResource() { +// This has to run before each test because we don't know when +// setupExtraResource() might be called +ResourceUtils.resetResourceTypes(new Configuration()); + } + + private static void setupExtraResource() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES, "test"); ResourceUtils.resetResourceTypes(conf); -ResourceUtils.getResourceTypes(); } public TestResourceCalculator(ResourceCalculator rs) { @@ -86,9 +91,15 @@ public class TestResourceCalculator { } } - private Resource newResource(long memory, int cpu, int test) { + private Resource newResource(long memory, int cpu) { Resource res = Resource.newInstance(memory, cpu); +return res; + } + + private Resource newResource(long memory, int cpu, int test) { +Resource res = newResource(memory, cpu); + res.setResourceValue("test", test); return res; @@ -123,28 +134,48 @@ public class TestResourceCalculator { } @Test - public void testCompare2() { + public void testCompareWithOnlyMandatory() { +// This test is necessary because there are optimizations that are only +// triggered when only the mandatory resources are configured. + // Keep cluster
[41/50] [abbrv] hadoop git commit: YARN-6610. [YARN-3926] DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources. Contributed by Daniel Templeton.
YARN-6610. [YARN-3926] DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources. Contributed by Daniel Templeton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84bebc86 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84bebc86 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84bebc86 Branch: refs/heads/YARN-3926 Commit: 84bebc867ec1dcb761bcb6bb1628748d71dba1e3 Parents: 40759d2 Author: Sunil GAuthored: Thu Aug 17 18:32:53 2017 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:50:35 2017 -0700 -- .../resource/DominantResourceCalculator.java| 275 --- .../util/resource/TestResourceCalculator.java | 226 --- 2 files changed, 429 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/84bebc86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 5992ba3..40b38b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -17,12 +17,18 @@ */ package org.apache.hadoop.yarn.util.resource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; /** * A {@link ResourceCalculator} which uses the concept of @@ -48,6 +54,7 @@ import org.apache.hadoop.yarn.util.UnitsConversionUtil; @Private @Unstable public class DominantResourceCalculator extends ResourceCalculator { + static final Log LOG = LogFactory.getLog(DominantResourceCalculator.class); public DominantResourceCalculator() { } @@ -92,7 +99,6 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public int compare(Resource clusterResource, Resource lhs, Resource rhs, boolean singleType) { - if (lhs.equals(rhs)) { return 0; } @@ -101,55 +107,232 @@ public class DominantResourceCalculator extends ResourceCalculator { return this.compare(lhs, rhs); } -float l = getResourceAsValue(clusterResource, lhs, true); -float r = getResourceAsValue(clusterResource, rhs, true); +// We have to calculate the shares for all resource types for both +// resources and then look for which resource has the biggest +// share overall. +ResourceInformation[] clusterRes = clusterResource.getResources(); +// If array creation shows up as a time sink, these arrays could be cached +// because they're always the same length. +double[] lhsShares = new double[clusterRes.length]; +double[] rhsShares = new double[clusterRes.length]; +double diff; + +try { + if (singleType) { +double[] max = new double[2]; + +calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares, max); + +diff = max[0] - max[1]; + } else if (clusterRes.length == 2) { +// Special case to handle the common scenario of only CPU and memory +// so the we can optimize for performance +diff = calculateSharesForMandatoryResources(clusterRes, lhs, rhs, +lhsShares, rhsShares); + } else { +calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares); + +Arrays.sort(lhsShares); +Arrays.sort(rhsShares); + +diff = compareShares(lhsShares, rhsShares); + } +} catch (ArrayIndexOutOfBoundsException ex) { + StringWriter out = new StringWriter(); // No need to close a StringWriter + ex.printStackTrace(new PrintWriter(out)); + + LOG.error("A problem was encountered while calculating resource " + + "availability that should not occur under normal circumstances. " + + "Please report this error to the Hadoop community by opening a " +
[03/50] [abbrv] hadoop git commit: YARN-6134. [ATSv2 Security] Regenerate delegation token for app just before token expires if app collector is active. Contributed by Varun Saxena
YARN-6134. [ATSv2 Security] Regenerate delegation token for app just before token expires if app collector is active. Contributed by Varun Saxena Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fd6ae24 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fd6ae24 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fd6ae24 Branch: refs/heads/YARN-3926 Commit: 7fd6ae24798cd3fdd77dbb00089a922407026e02 Parents: e276c75 Author: Jian HeAuthored: Fri Aug 18 23:20:44 2017 -0700 Committer: Varun Saxena Committed: Wed Aug 30 11:29:54 2017 +0530 -- .../security/TestTimelineAuthFilterForV2.java | 104 ++-- .../collector/AppLevelTimelineCollector.java| 35 -- .../collector/NodeTimelineCollectorManager.java | 118 ++- ...neV2DelegationTokenSecretManagerService.java | 11 ++ 4 files changed, 222 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fd6ae24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java index f1d5185..bc1594c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.security; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,6 +28,7 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -51,6 +53,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.KerberosTestUtils; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineV2Client; @@ -190,6 +194,10 @@ public class TestTimelineAuthFilterForV2 { // renewed automatically if app is still alive. conf.setLong( YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, 100); + // Set token max lifetime to 4 seconds to test if timeline delegation + // token for the app is regenerated automatically if app is still alive. + conf.setLong( + YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, 4000); } UserGroupInformation.setConfiguration(conf); collectorManager = new DummyNodeTimelineCollectorManager(); @@ -205,9 +213,8 @@ public class TestTimelineAuthFilterForV2 { if (!withKerberosLogin) { AppLevelTimelineCollector collector = (AppLevelTimelineCollector)collectorManager.get(appId); - org.apache.hadoop.security.token.Token - token = - collector.getDelegationTokenForApp(); + Token token = + collector.getDelegationTokenForApp(); token.setService(new Text("localhost" + token.getService().toString(). substring(token.getService().toString().indexOf(":"; UserGroupInformation.getCurrentUser().addToken(token); @@ -304,6 +311,20 @@ public class TestTimelineAuthFilterForV2 { } } + private boolean publishWithRetries(ApplicationId appId, File entityTypeDir, + String entityType, int numEntities) throws Exception { +for (int i = 0; i < 10; i++) { + try { +publishAndVerifyEntity(appId, entityTypeDir, entityType,
[15/50] [abbrv] hadoop git commit: YARN-5708. Implement APIs to get resource profiles from the RM. Contributed by Varun Vasudev.
YARN-5708. Implement APIs to get resource profiles from the RM. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75a175a0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75a175a0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75a175a0 Branch: refs/heads/YARN-3926 Commit: 75a175a0cfa4b224bfd657ecb441fc18773f5654 Parents: 071bcfb Author: Varun VasudevAuthored: Sat Oct 22 20:15:47 2016 +0530 Committer: Wangda Tan Committed: Wed Aug 30 15:45:13 2017 -0700 -- .../hadoop/mapred/ResourceMgrDelegate.java | 13 ++ .../hadoop/mapred/TestClientRedirect.java | 17 +++ .../yarn/api/ApplicationClientProtocol.java | 37 + .../GetAllResourceProfilesRequest.java | 35 + .../GetAllResourceProfilesResponse.java | 60 .../GetResourceProfileRequest.java | 59 .../GetResourceProfileResponse.java | 68 + .../yarn/api/records/ProfileCapability.java | 88 .../main/proto/applicationclient_protocol.proto | 2 + .../src/main/proto/yarn_protos.proto| 15 ++ .../src/main/proto/yarn_service_protos.proto| 16 +++ .../hadoop/yarn/client/api/YarnClient.java | 25 .../yarn/client/api/impl/YarnClientImpl.java| 19 +++ .../ApplicationClientProtocolPBClientImpl.java | 36 + .../ApplicationClientProtocolPBServiceImpl.java | 42 ++ .../pb/GetAllResourceProfilesRequestPBImpl.java | 55 +++ .../GetAllResourceProfilesResponsePBImpl.java | 142 +++ .../pb/GetResourceProfileRequestPBImpl.java | 101 + .../pb/GetResourceProfileResponsePBImpl.java| 112 +++ .../impl/pb/ProfileCapabilityPBImpl.java| 134 + .../hadoop/yarn/api/TestPBImplRecords.java | 34 + .../yarn/server/MockResourceManagerFacade.java | 16 +++ .../server/resourcemanager/ClientRMService.java | 41 ++ .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 12 ++ .../server/resourcemanager/ResourceManager.java | 9 ++ 26 files changed, 1193 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/75a175a0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 62aa497..a365f80 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -517,4 +518,16 @@ public class ResourceMgrDelegate extends YarnClient { throws YarnException, IOException { client.killApplication(appId, diagnostics); } + + @Override + public Map getResourceProfiles() + throws YarnException, IOException { +return client.getResourceProfiles(); + } + + @Override + public Resource getResourceProfile(String profile) + throws YarnException, IOException { +return client.getResourceProfile(profile); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75a175a0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 65eac65..cc50be0 100644 ---
[18/50] [abbrv] hadoop git commit: YARN-4081. Add support for multiple resource types in the Resource class. (Varun Vasudev via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37148ded/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt -- diff --git a/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt b/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 000..63fbc9d --- /dev/null +++ b/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,283 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom+* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni), +which has the following notices: +* This product includes software developed by FuseSource Corp. + http://fusesource.com +* This product includes software developed at + Progress Software Corporation and/or its subsidiaries or affiliates. +* This product includes software developed by IBM Corporation and others. + +The binary distribution of this product bundles binaries of +AWS Java SDK 1.10.6, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + +The Netty Project += + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +--- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: +* license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: +* http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ +* http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: +* license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: +* http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: +* license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: +* https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: +* license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: +* http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: +* license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: +* license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: +* http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: +* license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: +* https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library