[1/2] git commit: SLIDER-439 RM never fulfilled Slider AM's container request after NM died on a node where HRegionServer was running
Repository: incubator-slider
Updated Branches:
refs/heads/develop 714a335df -> ec07bdc12
SLIDER-439 RM never fulfilled Slider AM's container request after NM died on a
node where HRegionServer was running
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34b909a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34b909a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34b909a8
Branch: refs/heads/develop
Commit: 34b909a8edb551b6e9aa7f5ab2b3f6bd04f1b7c5
Parents: 6da903d
Author: Gour Saha
Authored: Wed Oct 22 21:19:52 2014 -0700
Committer: Gour Saha
Committed: Wed Oct 22 21:19:52 2014 -0700
--
.../server/appmaster/SliderAppMaster.java | 5 +-
.../slider/server/appmaster/state/AppState.java | 5 ++
.../server/appmaster/state/RoleHistory.java | 45 +++-
.../TestRoleHistoryContainerEvents.groovy | 55
4 files changed, 108 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34b909a8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 06d3597..b3c4b4c 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1514,7 +1514,10 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
*/
@Override //AMRMClientAsync
public void onNodesUpdated(List updatedNodes) {
-LOG_YARN.info("Nodes updated");
+LOG_YARN.info("onNodesUpdated({})", updatedNodes.size());
+log.info("Updated nodes {}", updatedNodes);
+// Check if any nodes are lost or revived and update state accordingly
+appState.onNodesUpdated(updatedNodes);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34b909a8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 706b0d2..db119bd 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -1242,6 +1243,10 @@ public class AppState {
}
}
+ public synchronized void onNodesUpdated(List updatedNodes) {
+roleHistory.onNodesUpdated(updatedNodes);
+ }
+
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34b909a8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index dca7384..9aca32f 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.common.tools.SliderUtils;
@@ -88,6 +89,14 @@ public class RoleHistory {
*/
private Map> availableNodes;
+ /**
+ * Track the failed nodes. Currently used to make wiser decision of container
+ * ask with/without locality. Has other potential uses as well.
+ */
+ private Map failedNodes = new HashMap();
+ // dummy to be used
git commit: SLIDER-557 ACL checks are stopping CI builds
Repository: incubator-slider
Updated Branches:
refs/heads/develop 6be4bfadc -> 3abc54fe5
SLIDER-557 ACL checks are stopping CI builds
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3abc54fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3abc54fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3abc54fe
Branch: refs/heads/develop
Commit: 3abc54fe51163d2110455901f4a67c3f6d080645
Parents: 6be4bfa
Author: Steve Loughran
Authored: Wed Oct 22 21:43:03 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 21:43:03 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 33 ++--
1 file changed, 23 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3abc54fe/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e7fa109..53a8b10 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -766,6 +766,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
amRegistrationData.getClientToAMTokenMasterKey().array());
applicationACLs = amRegistrationData.getApplicationACLs();
+// fix up the ACLs if they are not set
+String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+if (acls == null) {
+ getConfig().set(SliderXmlConfKeys.KEY_PROTOCOL_ACL, "*");
+}
//tell the server what the ACLs are
rpcService.getServer().refreshServiceAcl(serviceConf,
new SliderAMPolicyProvider());
@@ -1358,17 +1363,9 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
*/
private void startSliderRPCServer(AggregateConf instanceDefinition)
throws IOException, SliderException {
+verifyIPCAccess();
+
-// verify that if the cluster is authed, the ACLs are set.
-boolean authorization = getConfig().getBoolean(
-CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
-false);
-String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL);
-if (authorization && SliderUtils.isUnset(acls)) {
- throw new BadConfigException("Application has IPC authorization enabled
in " +
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION +
- " but no ACLs in " + SliderXmlConfKeys.KEY_PROTOCOL_ACL);
-}
SliderClusterProtocolPBImpl protobufRelay =
new SliderClusterProtocolPBImpl(this);
BlockingService blockingService = SliderClusterAPI.SliderClusterProtocolPB
@@ -1387,6 +1384,22 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
deployChildService(rpcService);
}
+ /**
+ * verify that if the cluster is authed, the ACLs are set.
+ * @throws BadConfigException if Authorization is set without any ACL
+ */
+ private void verifyIPCAccess() throws BadConfigException {
+boolean authorization = getConfig().getBoolean(
+CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+false);
+String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+if (authorization && SliderUtils.isUnset(acls)) {
+ throw new BadConfigException("Application has IPC authorization enabled
in " +
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION +
+ " but no ACLs in " + SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+}
+ }
+
/* === */
/* AMRMClientAsync callbacks */
git commit: SLIDER-505 enable port range specification for AM
Repository: incubator-slider
Updated Branches:
refs/heads/develop e13d20e3a -> 6be4bfadc
SLIDER-505 enable port range specification for AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6be4bfad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6be4bfad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6be4bfad
Branch: refs/heads/develop
Commit: 6be4bfadcec39f6912861277d74e0d8e0a46e603
Parents: e13d20e
Author: Jon Maron
Authored: Wed Oct 22 15:09:54 2014 -0400
Committer: Jon Maron
Committed: Wed Oct 22 15:09:54 2014 -0400
--
.../org/apache/slider/common/SliderKeys.java| 2 +
.../apache/slider/common/tools/PortScanner.java | 90 +++
.../server/appmaster/SliderAppMaster.java | 30 ++-
.../slider/common/tools/TestPortScan.groovy | 91 +++-
4 files changed, 208 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6be4bfad/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index e75ec73..ae58ef3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -182,4 +182,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String AM_FILTER_NAME =
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
+
+ String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6be4bfad/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
new file mode 100644
index 000..0f4cfbc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.tools;
+
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ *
+ */
+public class PortScanner {
+ private static Pattern NUMBER_RANGE =
Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
+ private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
+
+ private List remainingPortsToCheck;
+
+ public PortScanner() {
+ }
+
+ public void setPortRange(String input) {
+// first split based on commas
+Set inputPorts= new TreeSet();
+String[] ranges = input.split(",");
+for ( String range : ranges ) {
+ Matcher m = SINGLE_NUMBER.matcher(range.trim());
+ if (m.find()) {
+inputPorts.add(Integer.parseInt(m.group()));
+ } else {
+m = NUMBER_RANGE.matcher(range.trim());
+if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ for (int i = start; i < end + 1; i++) {
+inputPorts.add(i);
+ }
+}
+ }
+}
+this.remainingPortsToCheck = new ArrayList(inputPorts);
+ }
+
+ public List getRemainingPortsToCheck() {
+return remainingPortsToCheck;
+ }
+
+ public int getAvailablePort () throws SliderException{
+boolean found = false;
+int availablePort = -1;
+Iterator portsToCheck = this.remainingPortsToCheck.iterator();
+while (portsToCheck
[8/9] git commit: SLIDER-460 move all tests from scanning stdout to parsing the JSON from cluster status operations and examining it via the ClusterDescription class
SLIDER-460 move all tests from scanning stdout to parsing the JSON from cluster
status operations and examining it via the ClusterDescription class
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43749719
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43749719
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43749719
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 437497191e52fe1f93f9642ed549e90e431304a8
Parents: 26f0e4f
Author: Steve Loughran
Authored: Tue Oct 21 18:58:32 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:48:23 2014 +0100
--
.../framework/AgentCommandTestBase.groovy | 66 +++-
.../funtest/framework/CommandTestBase.groovy| 24 +++
.../funtest/lifecycle/AMFailuresIT.groovy | 61 +++---
.../funtest/lifecycle/AgentFailures2IT.groovy | 37 +++
.../funtest/lifecycle/AgentFailuresIT.groovy| 41 +++-
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 3 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 48 ++
7 files changed, 118 insertions(+), 162 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43749719/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 69523aa..effea44 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -128,25 +129,56 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
shell.dumpOutput();
}
- public static void assertComponentCount(String component, int count,
SliderShell shell) {
-log.info("Asserting component count.")
-int instanceCount = getComponentCount(component, shell)
-assert count == instanceCount, 'Instance count for component did not match
expected.'
- }
-
- public static int getComponentCount(String component, SliderShell shell) {
-String entry = findLineEntry(shell, ["instances", component] as String[])
-int instanceCount = 0
-if (!SliderUtils.isUnset(entry)) {
- log.info(entry)
- int index = entry.indexOf("container_")
- while (index != -1) {
-instanceCount++;
-index = entry.indexOf("container_", index + 1)
- }
+
+ public ClusterDescription execStatus(String application) {
+ClusterDescription cd
+File statusFile = File.createTempFile("status", ".json")
+try {
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ application,
+ ARG_OUTPUT, statusFile.absolutePath
+ ])
+
+ assert statusFile.exists()
+ cd = new ClusterDescription();
+ cd.fromFile(statusFile)
+ return cd
+} finally {
+ statusFile.delete()
}
+ }
-return instanceCount
+ public int queryRequestedCount(String application, String role) {
+ClusterDescription cd = execStatus(application)
+int requestedCount = cd.statistics[role]["containers.requested"]
+return requestedCount
+ }
+
+ boolean hasRequestedContainerCountExceeded(Map args) {
+String application = args['application']
+String role = args['role']
+int expectedCount = args['limit'].toInteger();
+return queryRequestedCount(application, role) >= expectedCount
+ }
+
+ public ClusterDescription expectContainersLive(String clustername,
+ String component,
+ int count) {
+ClusterDescription cd = execStatus(clustername)
+assertContainersLive(cd, component, count)
+return cd;
+ }
+
+ public static void assertContainersLive(ClusterDescription
clusterDescription,
+ String component, int count) {
+log.info("Asserting component count.")
+int instanceCount = clusterDescription.instances[component].size()
+if (count != instanceCount) {
+ log.warn(clusterDescription.toString())
+}
+assert count == instanceCount
}
public static String findLineEntry(SliderShell shell, String[] locater
[3/9] git commit: SLIDER-553 funtests to permit launching of slider.py on unix
SLIDER-553 funtests to permit launching of slider.py on unix
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/86639d21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/86639d21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/86639d21
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 86639d21b182ee1e0b8969301454ee226b01b9f3
Parents: 46e131d
Author: Steve Loughran
Authored: Wed Oct 22 10:31:08 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:31:08 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 5 +++--
.../slider/funtest/framework/CommandTestBase.groovy| 13 ++---
.../slider/funtest/framework/FuntestProperties.groovy | 5 +
.../slider/funtest/lifecycle/AgentRegistryIT.groovy| 5 ++---
4 files changed, 20 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index 94e40a2..c8ea0ce 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -51,8 +51,9 @@ to explain the code here
"""
def executeEnvSh(confDir):
- if not IS_WINDOWS:
-envCmd = 'source %s/slider-env.sh && env' % confDir
+ envscript = '%s/slider-env.sh' % confDir
+ if not IS_WINDOWS and os.path.exists(envscript):
+envCmd = 'source %s && env' % envscript
command = ['bash', '-c', envCmd]
proc = subprocess.Popen(command, stdout = subprocess.PIPE)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 58c61ea..9230a7a 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -87,8 +87,8 @@ abstract class CommandTestBase extends SliderTestUtils {
* shell-escaped ~ symbol. On windows this does
* not need to be escaped
*/
- static final String TILDE
-
+ public static final String TILDE
+
/*
Static initializer for test configurations. If this code throws exceptions
(which it may) the class will not be instantiable.
@@ -113,6 +113,8 @@ abstract class CommandTestBase extends SliderTestUtils {
TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed(
KEY_TEST_AM_KEYTAB)
+
+
TILDE = Shell.WINDOWS? "~" : "\\~"
}
@@ -126,7 +128,12 @@ abstract class CommandTestBase extends SliderTestUtils {
Configuration conf = loadSliderConf();
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.scriptFile = SliderShell.windows ? SLIDER_SCRIPT_PYTHON :
SLIDER_SCRIPT
+
+// choose python script if on windows or the launch key recommends it
+//
+boolean python = SLIDER_CONFIG.getBoolean(KEY_LAUNCH_PYTHON, false)
+SliderShell.scriptFile =
+(SliderShell.windows || python) ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
//set the property of the configuration directory
def path = SLIDER_CONF_DIRECTORY.absolutePath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 7d38ffc..92a7a95 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -74,4 +74,9 @@ public interface FuntestProperties extends
SliderXMLConfKeysForTesting {
String CORE_SITE_XML = "core-site.xml"
String HDFS_SITE_XML = "hdfs-site.xml"
String YARN_SITE_XML = "yarn-site.xml"
+
+ /**
+ * Flag to indicate that the .py script should be launched: {@value}
+ */
+ String KEY_LAUNCH_PYTHON = "slider.test.launch.python"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
---
[2/9] git commit: SLIDER-517 enable certificate and key localization for containers
SLIDER-517 enable certificate and key localization for containers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46e131df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46e131df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46e131df
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 46e131df617c2461ef88905d083a2802468f0494
Parents: 0fa1141
Author: Jon Maron
Authored: Tue Oct 21 14:24:47 2014 -0400
Committer: Jon Maron
Committed: Tue Oct 21 14:24:47 2014 -0400
--
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 155 +++
.../server/appmaster/SliderAppMaster.java | 30
.../appmaster/web/rest/agent/AgentWebApp.java | 4 +-
.../services/security/CertificateManager.java | 74 -
5 files changed, 229 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 2612865..9d5eb3d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -97,6 +97,8 @@ public interface AgentKeys {
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
String AGENT_OUT_FILE = "slider-agent.out";
+ String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
+ String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 19ec015..d0ae5be 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -20,18 +20,22 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -39,6 +43,7 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -88,6 +93,7 @@ import
org.apache.slider.server.appmaster.web.rest.agent.Register;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
+import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,7 +109,6 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
-import ja
[7/9] git commit: SLIDER-148 one final unit test on the exists command
SLIDER-148 one final unit test on the exists command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26f0e4f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26f0e4f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26f0e4f5
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 26f0e4f51d4d18a9f64c32830149f330ab1d83d0
Parents: e13d20e
Author: Steve Loughran
Authored: Tue Oct 21 18:57:00 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:48:23 2014 +0100
--
.../groovy/org/apache/slider/agent/actions/TestActionExists.groovy | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/26f0e4f5/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 2645e27..277fa7d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -124,6 +124,7 @@ class TestActionExists extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
+assert LauncherExitCodes.EXIT_FALSE ==
sliderClient.actionExists(clustername, args)
}
}
[3/4] git commit: SLIDER-550 custom JVM ops lose slider conf dir: bin/slider.py
SLIDER-550 custom JVM ops lose slider conf dir: bin/slider.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b48ad063
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b48ad063
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b48ad063
Branch: refs/heads/develop
Commit: b48ad06330c7c21543da9aeb736c580187256999
Parents: 1e51d6d
Author: Steve Loughran
Authored: Wed Oct 22 10:37:07 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:37:07 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b48ad063/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index c8ea0ce..a6dd90e 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -33,7 +33,8 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
HADOOP_CONF_DIR = "HADOOP_CONF_DIR"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
-DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m -Dslider.confdir=%s"
+SLIDER_CONFDIR_OPTS ="-Dslider.confdir=%s"
+DEFAULT_JVM_OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m"
ON_POSIX = 'posix' in sys.builtin_module_names
@@ -244,9 +245,10 @@ def main():
libdir = dirMustExist(libDir(slider_home))
confdir = dirMustExist(confDir(slider_home))
executeEnvSh(confdir)
- default_jvm_opts = DEFAULT_JVM__OPTS % confdir
+ jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split()
+ default_jvm_opts = DEFAULT_JVM_OPTS
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
- jvm_opts_split = slider_jvm_opts.split()
+ jvm_opts_list.extend(slider_jvm_opts.split())
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
hadoop_conf_dir = os.environ.get(HADOOP_CONF_DIR, "")
p = os.pathsep# path separator
@@ -257,14 +259,14 @@ def main():
+ hadoop_conf_dir
- #print "slider_home = \"%s\"" % slider_home
- #print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
- #print "slider_classpath = \"%s\"" % slider_classpath
+ debug("slider_home = \"%s\"" % slider_home)
+ debug("slider_jvm_opts = \"%s\"" % slider_jvm_opts)
+ debug("slider_classpath = \"%s\"" % slider_classpath)
return java(SLIDER_CLASSNAME,
args,
slider_classpath,
- jvm_opts_split)
+ jvm_opts_list)
if __name__ == '__main__':
"""
[2/4] git commit: SLIDER-550 custom JVM ops lose slider conf dir: bin/slider
SLIDER-550 custom JVM ops lose slider conf dir: bin/slider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1e51d6db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1e51d6db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1e51d6db
Branch: refs/heads/develop
Commit: 1e51d6db6132c3a21b3ee64dcfba07025a383908
Parents: 86639d2
Author: Steve Loughran
Authored: Wed Oct 22 10:36:32 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:36:32 2014 +0100
--
slider-assembly/src/main/scripts/slider | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1e51d6db/slider-assembly/src/main/scripts/slider
--
diff --git a/slider-assembly/src/main/scripts/slider
b/slider-assembly/src/main/scripts/slider
index 787016a..4228843 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -55,7 +55,8 @@ if [ -f "${confdir}/slider-env.sh" ]; then
. "${confdir}/slider-env.sh"
fi
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m -Dslider.confdir=${confdir}"
+slider_confdir_opts="-Dslider.confdir=${confdir}"
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m"
slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
# allow for an extra classpath
@@ -87,4 +88,4 @@ fi
#echo $javabin
#type ${javabin}java 2>&1
-exec ${javabin}java ${slider_jvm_opts} ${launcher} $@
+exec ${javabin}java ${slider_jvm_opts} ${slider_confdir_opts} ${launcher} $@
[1/4] git commit: SLIDER-553 funtests to permit launching of slider.py on unix
Repository: incubator-slider
Updated Branches:
refs/heads/develop 46e131df6 -> e13d20e3a
SLIDER-553 funtests to permit launching of slider.py on unix
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/86639d21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/86639d21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/86639d21
Branch: refs/heads/develop
Commit: 86639d21b182ee1e0b8969301454ee226b01b9f3
Parents: 46e131d
Author: Steve Loughran
Authored: Wed Oct 22 10:31:08 2014 +0100
Committer: Steve Loughran
Committed: Wed Oct 22 10:31:08 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 5 +++--
.../slider/funtest/framework/CommandTestBase.groovy| 13 ++---
.../slider/funtest/framework/FuntestProperties.groovy | 5 +
.../slider/funtest/lifecycle/AgentRegistryIT.groovy| 5 ++---
4 files changed, 20 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index 94e40a2..c8ea0ce 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -51,8 +51,9 @@ to explain the code here
"""
def executeEnvSh(confDir):
- if not IS_WINDOWS:
-envCmd = 'source %s/slider-env.sh && env' % confDir
+ envscript = '%s/slider-env.sh' % confDir
+ if not IS_WINDOWS and os.path.exists(envscript):
+envCmd = 'source %s && env' % envscript
command = ['bash', '-c', envCmd]
proc = subprocess.Popen(command, stdout = subprocess.PIPE)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 58c61ea..9230a7a 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -87,8 +87,8 @@ abstract class CommandTestBase extends SliderTestUtils {
* shell-escaped ~ symbol. On windows this does
* not need to be escaped
*/
- static final String TILDE
-
+ public static final String TILDE
+
/*
Static initializer for test configurations. If this code throws exceptions
(which it may) the class will not be instantiable.
@@ -113,6 +113,8 @@ abstract class CommandTestBase extends SliderTestUtils {
TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed(
KEY_TEST_AM_KEYTAB)
+
+
TILDE = Shell.WINDOWS? "~" : "\\~"
}
@@ -126,7 +128,12 @@ abstract class CommandTestBase extends SliderTestUtils {
Configuration conf = loadSliderConf();
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.scriptFile = SliderShell.windows ? SLIDER_SCRIPT_PYTHON :
SLIDER_SCRIPT
+
+// choose python script if on windows or the launch key recommends it
+//
+boolean python = SLIDER_CONFIG.getBoolean(KEY_LAUNCH_PYTHON, false)
+SliderShell.scriptFile =
+(SliderShell.windows || python) ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
//set the property of the configuration directory
def path = SLIDER_CONF_DIRECTORY.absolutePath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 7d38ffc..92a7a95 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -74,4 +74,9 @@ public interface FuntestProperties extends
SliderXMLConfKeysForTesting {
String CORE_SITE_XML = "core-site.xml"
String HDFS_SITE_XML = "hdfs-site.xml"
String YARN_SITE_XML = "yarn-site.xml"
+
+ /**
+ * Flag to indicate that the .py script should be launched: {@value}
+ */
+ String KEY_LAUNCH_PYTHON = "slider.test.launch.python"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/86639d21/slider-funtest/src/test/groovy/org
[4/4] git commit: Merge branch 'feature/SLIDER-550_custom_JVM_ops_lose_slider_conf_dir' into develop
Merge branch 'feature/SLIDER-550_custom_JVM_ops_lose_slider_conf_dir' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e13d20e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e13d20e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e13d20e3 Branch: refs/heads/develop Commit: e13d20e3a31b26e1353f87c8c69ab0cf0efd8746 Parents: 46e131d b48ad06 Author: Steve Loughran Authored: Wed Oct 22 10:37:15 2014 +0100 Committer: Steve Loughran Committed: Wed Oct 22 10:37:15 2014 +0100 -- slider-assembly/src/main/scripts/slider | 5 +++-- slider-assembly/src/main/scripts/slider.py | 21 +++- .../funtest/framework/CommandTestBase.groovy| 13 +--- .../funtest/framework/FuntestProperties.groovy | 5 + .../funtest/lifecycle/AgentRegistryIT.groovy| 5 ++--- 5 files changed, 32 insertions(+), 17 deletions(-) --
svn commit: r926423 - in /websites/staging/slider/trunk/content: ./ developing/functional_tests.html
Author: buildbot Date: Wed Oct 22 09:31:28 2014 New Revision: 926423 Log: Staging update by buildbot for slider Modified: websites/staging/slider/trunk/content/ (props changed) websites/staging/slider/trunk/content/developing/functional_tests.html Propchange: websites/staging/slider/trunk/content/ -- --- cms:source-revision (original) +++ cms:source-revision Wed Oct 22 09:31:28 2014 @@ -1 +1 @@ -1633372 +1633573 Modified: websites/staging/slider/trunk/content/developing/functional_tests.html == --- websites/staging/slider/trunk/content/developing/functional_tests.html (original) +++ websites/staging/slider/trunk/content/developing/functional_tests.html Wed Oct 22 09:31:28 2014 @@ -296,8 +296,10 @@ need to be changed@@ -306,6 +308,15 @@ need to be changed slider.test.timeout.millisec -Time out in milliseconds before a test is considered to have failed. - There are some maven properties which also define limits and may need adjusting ++ Time out in milliseconds before a test is considered to have failed. + There are some maven properties which also define limits and may need adjusting + 18 Size in MB to ask for containers 192 + ++ slider.test.launch.python ++ Flag to mandate use of the slider.py script. + Ignored on windows âwhich always uses the python script + +false +
svn commit: r1633573 - /incubator/slider/site/trunk/content/developing/functional_tests.md
Author: stevel Date: Wed Oct 22 09:31:22 2014 New Revision: 1633573 URL: http://svn.apache.org/r1633573 Log: SLIDER-553 funtests to permit launching of slider.py on unix Modified: incubator/slider/site/trunk/content/developing/functional_tests.md Modified: incubator/slider/site/trunk/content/developing/functional_tests.md URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/functional_tests.md?rev=1633573&r1=1633572&r2=1633573&view=diff == --- incubator/slider/site/trunk/content/developing/functional_tests.md (original) +++ incubator/slider/site/trunk/content/developing/functional_tests.md Wed Oct 22 09:31:22 2014 @@ -148,8 +148,10 @@ need to be changed slider.test.timeout.millisec - Time out in milliseconds before a test is considered to have failed. - There are some maven properties which also define limits and may need adjusting + + Time out in milliseconds before a test is considered to have failed. + There are some maven properties which also define limits and may need adjusting + 18 @@ -159,6 +161,15 @@ need to be changed 192 + + slider.test.launch.python + + Flag to mandate use of the slider.py script. + Ignored on windows âwhich always uses the python script + + false + + Note that while the same properties need to be set in `slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
