[11/50] git commit: SLIDER-352 some tests
SLIDER-352 some tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/83665f61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/83665f61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/83665f61
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 83665f617025fea9f8f7e5462e30d66caab025d0
Parents: b3a835d
Author: Steve Loughran
Authored: Fri Aug 22 16:08:24 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 22 16:08:24 2014 +0100
--
.../common/tools/TestWindowsSupport.groovy | 78
1 file changed, 78 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/83665f61/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
new file mode 100644
index 000..4da42f2
--- /dev/null
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.ChecksumFileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.junit.Test
+
+import java.util.regex.Pattern
+
+@CompileStatic
+@Slf4j
+class TestWindowsSupport {
+
+ private static final Pattern hasDriveLetterSpecifier =
+ Pattern.compile("^/?[a-zA-Z]:");
+ public static
+ final String windowsFile =
"C:\\Users\\Administrator\\AppData\\Local\\Temp\\junit3180177850133852404\\testpkg\\appdef_1.zip"
+
+
+ private static boolean hasWindowsDrive(String path) {
+return hasDriveLetterSpecifier.matcher(path).find();
+ }
+
+ private static int startPositionWithoutWindowsDrive(String path) {
+if (hasWindowsDrive(path)) {
+ return path.charAt(0) == '/' ? 3 : 2;
+} else {
+ return 0;
+}
+ }
+
+ @Test
+ public void testHasWindowsDrive() throws Throwable {
+assert hasWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testStartPosition() throws Throwable {
+assert 2 == startPositionWithoutWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testPathHandling() throws Throwable {
+System.setProperty("os.name", "Windows/286")
+Path path = new Path(windowsFile);
+def uri = path.toUri()
+//assert "file" == uri.scheme
+assert uri.authority == null;
+
+
+
+Configuration conf = new Configuration()
+
+def localfs = HadoopFS.get(uri, conf)
+assert localfs instanceof ChecksumFileSystem
+ }
+}
[29/50] git commit: Merge branch 'feature/SLIDER-353_killJavaProcesses_doesn't_work_on_windows' into develop
Merge branch 'feature/SLIDER-353_killJavaProcesses_doesn't_work_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7b7d29e9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7b7d29e9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7b7d29e9 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 7b7d29e9a0327d3108d57317896518ddefd677dd Parents: d8f8771 38c2a13 Author: Steve Loughran Authored: Mon Aug 25 13:23:11 2014 +0100 Committer: Steve Loughran Committed: Mon Aug 25 13:23:11 2014 +0100 -- .../common/tools/TestWindowsSupport.groovy | 6 +++ .../apache/slider/test/SliderTestUtils.groovy | 39 .../slider/test/YarnMiniClusterTestBase.groovy | 23 3 files changed, 45 insertions(+), 23 deletions(-) --
[13/50] git commit: SLIDER-352 more testing of windows
SLIDER-352 more testing of windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/53457d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/53457d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/53457d98
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 53457d98d480e3b4134175c09b4927d0c1dfb4ef
Parents: db2d298
Author: Steve Loughran
Authored: Fri Aug 22 16:20:26 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 22 16:20:26 2014 +0100
--
.../apache/slider/providers/agent/AgentUtils.java| 9 +++--
.../slider/common/tools/TestWindowsSupport.groovy| 15 +--
2 files changed, 20 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53457d98/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index f296a95..f329756 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -24,6 +25,7 @@ import
org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -36,11 +38,14 @@ public class AgentUtils {
static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String appDef) throws IOException {
log.info("Reading metainfo at {}", appDef);
+FileSystem fs = fileSystem.getFileSystem();
+Path appPath = new Path(appDef);
InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
-fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
+fs, appPath, "metainfo.xml");
if (metainfoStream == null) {
log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new IOException("metainfo.xml is required in app package.");
+ throw new FileNotFoundException("metainfo.xml is required in app
package. " +
+appPath);
}
Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53457d98/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 8387237..2d74760 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.ChecksumFileSystem
+import org.apache.hadoop.fs.FSDataInputStream
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
@@ -75,7 +76,17 @@ class TestWindowsSupport extends SliderTestUtils {
def localfs = HadoopFS.get(uri, conf)
assert localfs instanceof ChecksumFileSystem
-def stat = localfs.getFileStatus(path)
-
+try {
+ def stat = localfs.getFileStatus(path)
+ fail("expected an exception, got $stat")
+} catch (FileNotFoundException fnfe) {
+ // expected
+}
+
+try {
+ FSDataInputStream appStream = localfs.open(path);
+} catch (FileNotFoundException fnfe) {
+ // expected
+}
}
}
[08/50] git commit: SLIDER-350: AM not coming up on Windows
SLIDER-350: AM not coming up on Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a5df3975
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a5df3975
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a5df3975
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a5df39755bfd044f1d2ee4cc49e8d96f80a855d2
Parents: 4e8c6ff
Author: Steve Loughran
Authored: Fri Aug 22 14:32:39 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 22 14:32:39 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 2 +-
.../server/appmaster/web/AgentService.java | 28 +---
.../appmaster/web/rest/agent/AgentWebApp.java | 16 ---
.../LaunchedWorkflowCompositeService.java | 4 +++
.../services/workflow/ClosingService.java | 3 +++
.../web/rest/agent/TestAMAgentWebServices.java | 2 +-
.../workflow/TestWorkflowClosingService.java| 4 +--
7 files changed, 29 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index c006d0d..b4515f1 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -813,7 +813,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
}
private void startAgentWebApp(MapOperations appInformation,
-Configuration serviceConf) {
+Configuration serviceConf) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader()
).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
index 08338e8..f840035 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
@@ -16,38 +16,22 @@
*/
package org.apache.slider.server.appmaster.web;
-import org.apache.hadoop.service.AbstractService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
+import org.apache.slider.server.services.workflow.ClosingService;
+import org.apache.slider.server.services.workflow.WorkflowCompositeService;
/**
- *
+ * agent service gives the agent webapp lifecycle integration
*/
-public class AgentService extends AbstractService {
- private volatile AgentWebApp webApp;
+public class AgentService extends ClosingService {
+
public AgentService(String name) {
super(name);
}
public AgentService(String name, AgentWebApp app) {
-super(name);
-webApp = app;
+super(name, app);
}
- @Override
- protected void serviceStart() throws Exception {
-
- }
-
- /**
- * Stop operation stops the webapp; sets the reference to null
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
-if (webApp != null) {
- webApp.stop();
- webApp = null;
-}
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a5df3975/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 54d2b1f..8aac490 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -36,13 +36,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.ext.Provider;
+import java.io.Closeable;
import java.io.File;
+import java.io.IOException;
import java.util.Set;
/**
*
*/
-public class AgentWebApp {
+public class AgentWebApp implements Clos
[15/50] git commit: SLIDER-352 TestPublisherRestResources failing on windows
SLIDER-352 TestPublisherRestResources failing on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4ca25b2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4ca25b2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4ca25b2c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4ca25b2c535bd8ebbb8d92b318436dffe778754f
Parents: 475507b
Author: Steve Loughran
Authored: Fri Aug 22 16:33:07 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 22 16:33:07 2014 +0100
--
.../web/rest/publisher/TestPublisherRestResources.groovy | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4ca25b2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index c1732e6..7dbd3e2 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -65,14 +65,15 @@ class TestPublisherRestResources extends AgentTestBase {
assert app_def_path.exists()
assert agt_ver_path.exists()
assert agt_conf_path.exists()
+
ServiceLauncher launcher = buildAgentCluster(clustername,
roles,
[
ARG_PROVIDER,
"org.apache.slider.server.appmaster.web.rest.publisher.TestSliderProviderFactory",
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath
+ARG_OPTION, APP_DEF, toFileURI(app_def_path),
+ARG_OPTION, AGENT_CONF, toFileURI(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toFileURI(agt_ver_path)
],
true, true,
true)
@@ -142,4 +143,8 @@ class TestPublisherRestResources extends AgentTestBase {
assert !uris.any {it =~ /foo-bar/}
}
+ public String toFileURI(File filename) {
+"file://" + filename.absolutePath
+ }
+
}
[36/50] git commit: SLIDER-254 update feature branch changes for java 6 compatibility
SLIDER-254 update feature branch changes for java 6 compatibility
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/621d2cd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/621d2cd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/621d2cd4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 621d2cd474f01190462b26da28caaa16bedbe9cc
Parents: c89b264
Author: Billie Rinaldi
Authored: Mon Aug 25 15:02:24 2014 -0700
Committer: Billie Rinaldi
Committed: Mon Aug 25 15:02:24 2014 -0700
--
.../src/main/java/org/apache/slider/client/SliderClient.java | 8
.../src/main/java/org/apache/slider/core/conf/ConfTree.java | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/621d2cd4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1a372e7..2107838 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -483,7 +483,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
CredentialProvider credentialProvider =
CredentialProviderFactory.getProviders(c).get(0);
- Set existingAliases = new
HashSet<>(credentialProvider.getAliases());
+ Set existingAliases = new
HashSet(credentialProvider.getAliases());
for (String alias : aliases) {
if (existingAliases.contains(alias.toLowerCase())) {
log.warn("Skipping creation of credentials for {}, " +
@@ -710,16 +710,16 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public static void replaceTokens(ConfTree conf,
String userName, String clusterName) throws IOException {
-Map newglobal = new HashMap<>();
+Map newglobal = new HashMap();
for (Entry entry : conf.global.entrySet()) {
newglobal.put(entry.getKey(), replaceTokens(entry.getValue(),
userName, clusterName));
}
conf.global.putAll(newglobal);
-Map> newcred = new HashMap<>();
+Map> newcred = new HashMap>();
for (Entry> entry : conf.credentials.entrySet()) {
- List resultList = new ArrayList<>();
+ List resultList = new ArrayList();
for (String v : entry.getValue()) {
resultList.add(replaceTokens(v, userName, clusterName));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/621d2cd4/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index be7c56f..2af8d26 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -70,7 +70,7 @@ public final class ConfTree {
* Credentials
*/
public Map> credentials =
- new HashMap<>(INITAL_MAP_CAPACITY);
+ new HashMap>(INITAL_MAP_CAPACITY);
/**
* Role options,
[25/50] git commit: SLIDER-357 slider bash script cannot handle if java is in the path
SLIDER-357 slider bash script cannot handle if java is in the path
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d8f8771b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d8f8771b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d8f8771b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d8f8771b9671b3b9ef97d2892c56d59c16ff6621
Parents: 41599dc
Author: tedyu
Authored: Sat Aug 23 21:09:38 2014 -0700
Committer: tedyu
Committed: Sat Aug 23 21:09:38 2014 -0700
--
slider-assembly/src/main/scripts/slider | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8f8771b/slider-assembly/src/main/scripts/slider
--
diff --git a/slider-assembly/src/main/scripts/slider
b/slider-assembly/src/main/scripts/slider
index 7172917..6cd55ad 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -71,5 +71,5 @@ echo ""
echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\"
${launcher} $@"
echo ""
echo ""
-type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin
-exec ${javabin}/java ${slider_jvm_opts} ${launcher} $@
+type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin/
+exec ${javabin}java ${slider_jvm_opts} ${launcher} $@
[06/50] git commit: Merge branch 'release/slider-0.50.2' into develop
Merge branch 'release/slider-0.50.2' into develop Conflicts: app-packages/accumulo/pom.xml app-packages/command-logger/application-pkg/pom.xml app-packages/command-logger/slider-pkg/pom.xml pom.xml slider-agent/pom.xml slider-assembly/pom.xml slider-core/pom.xml slider-funtest/pom.xml slider-providers/accumulo/accumulo-funtests/pom.xml slider-providers/accumulo/slider-accumulo-provider/pom.xml slider-providers/hbase/hbase-funtests/pom.xml slider-providers/hbase/slider-hbase-provider/pom.xml Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1e0d1e80 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1e0d1e80 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1e0d1e80 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 1e0d1e80f19d3e6a1a6e08c7936ff858965f6fae Parents: afaa8e4 bc6c710 Author: Steve Loughran Authored: Fri Aug 22 12:23:58 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 22 12:23:58 2014 +0100 -- app-packages/accumulo/pom.xml | 2 +- app-packages/command-logger/application-pkg/pom.xml| 2 +- app-packages/command-logger/slider-pkg/pom.xml | 2 +- pom.xml| 2 +- slider-agent/pom.xml | 2 +- slider-assembly/pom.xml| 2 +- slider-core/pom.xml| 2 +- slider-funtest/pom.xml | 2 +- slider-providers/accumulo/accumulo-funtests/pom.xml| 2 +- slider-providers/accumulo/slider-accumulo-provider/pom.xml | 2 +- slider-providers/hbase/hbase-funtests/pom.xml | 2 +- slider-providers/hbase/slider-hbase-provider/pom.xml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) --
[05/50] git commit: SLIDER-349 updating release POMs for 0.50.2-incubating
SLIDER-349 updating release POMs for 0.50.2-incubating Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc6c7103 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc6c7103 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc6c7103 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: bc6c7103353aea9c96656443c5dd3a292806a1b7 Parents: d8a1114 Author: Steve Loughran Authored: Fri Aug 22 11:56:18 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 22 11:56:18 2014 +0100 -- app-packages/accumulo/pom.xml | 2 +- app-packages/command-logger/application-pkg/pom.xml| 2 +- app-packages/command-logger/slider-pkg/pom.xml | 2 +- pom.xml| 2 +- slider-agent/pom.xml | 2 +- slider-assembly/pom.xml| 2 +- slider-core/pom.xml| 2 +- slider-funtest/pom.xml | 2 +- slider-providers/accumulo/accumulo-funtests/pom.xml| 2 +- slider-providers/accumulo/slider-accumulo-provider/pom.xml | 2 +- slider-providers/hbase/hbase-funtests/pom.xml | 2 +- slider-providers/hbase/slider-hbase-provider/pom.xml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/app-packages/accumulo/pom.xml -- diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml index 88ea222..fe71c70 100644 --- a/app-packages/accumulo/pom.xml +++ b/app-packages/accumulo/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating ../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/app-packages/command-logger/application-pkg/pom.xml -- diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml index 1b572b3..71e4d82 100644 --- a/app-packages/command-logger/application-pkg/pom.xml +++ b/app-packages/command-logger/application-pkg/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating ../../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/app-packages/command-logger/slider-pkg/pom.xml -- diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml index fe485ee..bd46cbb 100644 --- a/app-packages/command-logger/slider-pkg/pom.xml +++ b/app-packages/command-logger/slider-pkg/pom.xml @@ -20,7 +20,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating ../../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/pom.xml -- diff --git a/pom.xml b/pom.xml index bbc94b9..fb27aba 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider Slider - 0.50.1-incubating + 0.50.2-incubating pom http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/slider-agent/pom.xml -- diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml index 9e4b71f..09f2dae 100644 --- a/slider-agent/pom.xml +++ b/slider-agent/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating 4.0.0 slider-agent http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/slider-assembly/pom.xml -- diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml index 6e69310..928713e 100644 --- a/slider-assembly/pom.xml +++ b/slider-assembly/pom.xml @@ -23,7 +23,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/slider-core/pom.xml -- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 4507480..b228874 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -23,7 +23,7 @@ org.apache.slider slider -0.50.1-incubating +0.50.2-incubating http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc6c7103/slider-funt
[26/50] git commit: SLIDER-353 killJavaProcesses doesn't work on windows: testcase
SLIDER-353 killJavaProcesses doesn't work on windows: testcase
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/73389fd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/73389fd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/73389fd5
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 73389fd57e36e2f5987161594c343009a618cebe
Parents: d8f8771
Author: Steve Loughran
Authored: Mon Aug 25 12:30:37 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 25 12:30:37 2014 +0100
--
.../common/tools/TestWindowsSupport.groovy | 6
.../apache/slider/test/SliderTestUtils.groovy | 29
.../slider/test/YarnMiniClusterTestBase.groovy | 23
3 files changed, 35 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73389fd5/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index d11aa55..7f5edf3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -103,4 +103,10 @@ class TestWindowsSupport extends SliderTestUtils {
}
}
+
+ @Test
+ public void testEmitKillCommand() throws Throwable {
+killJavaProcesses("regionserver", 9)
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73389fd5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 3fc3e55..0a1354b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ClusterDescription
@@ -737,4 +738,32 @@ class SliderTestUtils extends Assert {
log.info "$key -- ${config.description}"
}
}
+
+ /**
+ * Kill any java process with the given grep pattern
+ * @param grepString string to grep for
+ */
+ public void killJavaProcesses(String grepString, int signal) {
+
+assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows
support ")
+
+GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
+log.info("Bash command = $bashCommand" )
+Process bash = ["bash", "-c", bashCommand].execute()
+bash.waitFor()
+
+log.info(bash.in.text)
+log.error(bash.err.text)
+ }
+
+ /**
+ * Kill all processes which match one of the list of grepstrings
+ * @param greps
+ * @param signal
+ */
+ public void killJavaProcesses(List greps, int signal) {
+for (String grep : greps) {
+ killJavaProcesses(grep,signal)
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73389fd5/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 09c56a5..5a091bf 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -322,29 +322,6 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
}
/**
- * Kill any java process with the given grep pattern
- * @param grepString string to grep for
- */
- public void killJavaProcesses(String grepString, int signal) {
-
-assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows
support ")
-
-GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
-log.info("Bash command = $bashCommand" )
-Process bash = ["bash", "-c", bashCommand].execute()
-b
[18/50] git commit: SLIDER-254 fix some issues with accumulo process startup
SLIDER-254 fix some issues with accumulo process startup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/31784988
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/31784988
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/31784988
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 31784988d67dfd5a9ea02e0d3faf15e82a0d152d
Parents: 5510e13
Author: Billie Rinaldi
Authored: Fri Aug 22 10:39:43 2014 -0700
Committer: Billie Rinaldi
Committed: Fri Aug 22 10:39:43 2014 -0700
--
app-packages/accumulo/package/scripts/accumulo_service.py | 2 +-
app-packages/accumulo/package/scripts/params.py| 2 +-
app-packages/accumulo/package/templates/accumulo-env.sh.j2 | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31784988/app-packages/accumulo/package/scripts/accumulo_service.py
--
diff --git a/app-packages/accumulo/package/scripts/accumulo_service.py
b/app-packages/accumulo/package/scripts/accumulo_service.py
index 562ef5d..ca21cc8 100644
--- a/app-packages/accumulo/package/scripts/accumulo_service.py
+++ b/app-packages/accumulo/package/scripts/accumulo_service.py
@@ -30,7 +30,7 @@ def accumulo_service( name,
pid_exists = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}`
>/dev/null 2>&1")
if action == 'start':
- daemon_cmd = format("{daemon_script} {role} >
{log_dir}/accumulo-{accumulo_user}-{role}.out
2>{log_dir}/accumulo-{accumulo_user}-{role}.err & echo $! > {pid_file}")
+ daemon_cmd = format("{daemon_script} {role} --address {params.hostname}
> {log_dir}/accumulo-{accumulo_user}-{role}.out
2>{log_dir}/accumulo-{accumulo_user}-{role}.err & echo $! > {pid_file}")
Execute ( daemon_cmd,
not_if=pid_exists,
user=params.accumulo_user
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31784988/app-packages/accumulo/package/scripts/params.py
--
diff --git a/app-packages/accumulo/package/scripts/params.py
b/app-packages/accumulo/package/scripts/params.py
index 5cef7eb..7d0e52d 100644
--- a/app-packages/accumulo/package/scripts/params.py
+++ b/app-packages/accumulo/package/scripts/params.py
@@ -23,7 +23,7 @@ import status_params
# server configurations
config = Script.get_config()
-hostname = config["hostname"]
+hostname = config["public_hostname"]
# user and status
accumulo_user = status_params.accumulo_user
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31784988/app-packages/accumulo/package/templates/accumulo-env.sh.j2
--
diff --git a/app-packages/accumulo/package/templates/accumulo-env.sh.j2
b/app-packages/accumulo/package/templates/accumulo-env.sh.j2
index 7ffec53..9e365af 100755
--- a/app-packages/accumulo/package/templates/accumulo-env.sh.j2
+++ b/app-packages/accumulo/package/templates/accumulo-env.sh.j2
@@ -36,7 +36,7 @@ export ACCUMULO_TSERVER_OPTS="-Xmx{{tserver_heapsize}}
-Xms{{tserver_heapsize}}"
export ACCUMULO_MASTER_OPTS="-Xmx{{master_heapsize}} -Xms{{master_heapsize}}"
export ACCUMULO_MONITOR_OPTS="-Xmx{{monitor_heapsize}}
-Xms{{monitor_heapsize}}"
export ACCUMULO_GC_OPTS="-Xmx{{gc_heapsize}} -Xms{{gc_heapsize}}"
-export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75"
+export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
export ACCUMULO_OTHER_OPTS="-Xmx{{other_heapsize}} -Xms{{other_heapsize}}"
# what do when the JVM runs out of heap memory
export ACCUMULO_KILL_CMD='kill -9 %p'
[07/50] git commit: SLIDER-349 reverting development POMs to 0.51.0-incubating-SNAPSHOT
SLIDER-349 reverting development POMs to 0.51.0-incubating-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4e8c6ffb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4e8c6ffb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4e8c6ffb Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4e8c6ffbb5a759c27b962485cb63592cfc2d87bf Parents: 1e0d1e8 Author: Steve Loughran Authored: Fri Aug 22 12:27:49 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 22 12:30:21 2014 +0100 -- app-packages/accumulo/pom.xml | 2 +- app-packages/command-logger/application-pkg/pom.xml| 2 +- app-packages/command-logger/slider-pkg/pom.xml | 2 +- pom.xml| 2 +- slider-agent/pom.xml | 2 +- slider-assembly/pom.xml| 2 +- slider-core/pom.xml| 2 +- slider-funtest/pom.xml | 2 +- slider-providers/accumulo/accumulo-funtests/pom.xml| 2 +- slider-providers/accumulo/slider-accumulo-provider/pom.xml | 2 +- slider-providers/hbase/hbase-funtests/pom.xml | 2 +- slider-providers/hbase/slider-hbase-provider/pom.xml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/app-packages/accumulo/pom.xml -- diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml index fe71c70..84be933 100644 --- a/app-packages/accumulo/pom.xml +++ b/app-packages/accumulo/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT ../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/app-packages/command-logger/application-pkg/pom.xml -- diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml index 71e4d82..77f0fc9 100644 --- a/app-packages/command-logger/application-pkg/pom.xml +++ b/app-packages/command-logger/application-pkg/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT ../../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/app-packages/command-logger/slider-pkg/pom.xml -- diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml index bd46cbb..b9bbf06 100644 --- a/app-packages/command-logger/slider-pkg/pom.xml +++ b/app-packages/command-logger/slider-pkg/pom.xml @@ -20,7 +20,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT ../../../pom.xml 4.0.0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/pom.xml -- diff --git a/pom.xml b/pom.xml index fb27aba..29c3c16 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider Slider - 0.50.2-incubating + 0.51.0-incubating-SNAPSHOT pom http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/slider-agent/pom.xml -- diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml index 09f2dae..f26cb11 100644 --- a/slider-agent/pom.xml +++ b/slider-agent/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT 4.0.0 slider-agent http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/slider-assembly/pom.xml -- diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml index 928713e..6c047e7 100644 --- a/slider-assembly/pom.xml +++ b/slider-assembly/pom.xml @@ -23,7 +23,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e8c6ffb/slider-core/pom.xml -- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index b228874..cdcea38 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -23,7 +23,7 @@ org.apache.slider slider -0.50.2-incubating +0.51.0-incubating-SNAPSHOT http:
[35/50] git commit: SLIDER-360. Accumulo package fails to deploy as Slider expects config files to be supported to be specified in the metainfo
SLIDER-360. Accumulo package fails to deploy as Slider expects config files to
be supported to be specified in the metainfo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1b9749c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1b9749c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1b9749c2
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1b9749c2dc5436cf46bfec33983fb8158c943600
Parents: 32f1b6b
Author: Sumit Mohanty
Authored: Mon Aug 25 14:10:03 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Aug 25 14:10:10 2014 -0700
--
app-packages/accumulo/appConfig.json | 13 -
app-packages/accumulo/metainfo.xml | 8
2 files changed, 8 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1b9749c2/app-packages/accumulo/appConfig.json
--
diff --git a/app-packages/accumulo/appConfig.json
b/app-packages/accumulo/appConfig.json
index 6b7033e..e9ca2ba 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -4,14 +4,11 @@
},
"global": {
"application.def": "${app.package.name}.zip",
-"config_types": "accumulo-site",
"java_home": "/usr/jdk64/jdk1.7.0_45",
-"package_list": "files/accumulo-${accumulo.version}-bin.tar.gz",
"site.global.app_user": "yarn",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
-"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
"site.global.tserver_heapsize": "128m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
@@ -44,18 +41,8 @@
"site.accumulo-site.general.classpaths":
"$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"components": {
-"ACCUMULO_MASTER": {
-},
"slider-appmaster": {
"jvm.heapsize": "256M"
-},
-"ACCUMULO_TSERVER": {
-},
-"ACCUMULO_MONITOR": {
-},
-"ACCUMULO_GC": {
-},
-"ACCUMULO_TRACER": {
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1b9749c2/app-packages/accumulo/metainfo.xml
--
diff --git a/app-packages/accumulo/metainfo.xml
b/app-packages/accumulo/metainfo.xml
index b1aa9de..512ba4a 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -144,5 +144,13 @@
+
+
+xml
+accumulo-site.xml
+accumulo-site
+
+
+
[28/50] git commit: SLIDER-353 killJavaProcesses doesn't work on windows
SLIDER-353 killJavaProcesses doesn't work on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38c2a138
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38c2a138
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38c2a138
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 38c2a138b54d9a87794446447a19707f0a51c4d4
Parents: 1d62a81
Author: Steve Loughran
Authored: Mon Aug 25 13:01:54 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 25 13:01:54 2014 +0100
--
.../apache/slider/test/SliderTestUtils.groovy | 30 +---
1 file changed, 20 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c2a138/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 8057356..0036db7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -743,17 +743,27 @@ class SliderTestUtils extends Assert {
* Kill any java process with the given grep pattern
* @param grepString string to grep for
*/
- public void killJavaProcesses(String grepString, int signal) {
-
-//assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows
support ")
-
-GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
-log.info("Bash command = $bashCommand" )
-Process bash = ["bash", "-c", bashCommand].execute()
-bash.waitFor()
+ public int killJavaProcesses(String grepString, int signal) {
+
+def commandString
+if (!Shell.WINDOWS) {
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
+ log.info("Command command = $killCommand" )
+
+ commandString = ["bash", "-c", killCommand]
+} else {
+ /*
+ "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
+ */
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | awk \"{print
\$1}\" | xargs -n 1 taskkill /PID\""
+ commandString = ["CMD", "/C", killCommand]
+}
+Process command = commandString.execute()
+def exitCode = command.waitFor()
-log.info(bash.in.text)
-log.error(bash.err.text)
+log.info(command.in.text)
+log.error(command.err.text)
+return exitCode
}
/**
[34/50] git commit: SLIDER-334 removed java.nio.file code from some tests for JDK 6 compatability
SLIDER-334 removed java.nio.file code from some tests for JDK 6 compatability
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/32f1b6b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/32f1b6b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/32f1b6b8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 32f1b6b84ec227d85dc979fdc20161a93c4703e9
Parents: bc9766b
Author: Jon Maron
Authored: Mon Aug 25 15:41:25 2014 -0400
Committer: Jon Maron
Committed: Mon Aug 25 15:41:25 2014 -0400
--
.../apache/slider/test/SliderTestBase.groovy| 2 +-
.../web/rest/agent/TestAMAgentWebServices.java | 48 ++--
2 files changed, 26 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32f1b6b8/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index 28b484f..afb07d8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -27,7 +27,7 @@ import org.junit.BeforeClass
import org.junit.Rule
import org.junit.rules.TestName
-import java.nio.file.Files
+//import java.nio.file.Files
/**
* Base class for unit tests as well as ones starting mini clusters
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32f1b6b8/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 827b47a..c825fb8 100644
---
a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++
b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -25,6 +25,7 @@ import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.IOUtils;
@@ -55,12 +56,12 @@ import javax.ws.rs.core.MediaType;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
+//import java.nio.file.FileVisitResult;
+//import java.nio.file.Files;
+//import java.nio.file.Path;
+//import java.nio.file.Paths;
+//import java.nio.file.SimpleFileVisitor;
+//import java.nio.file.attribute.BasicFileAttributes;
import static org.junit.Assert.assertEquals;
@@ -219,22 +220,23 @@ public class TestAMAgentWebServices {
@AfterClass
public static void tearDownClass() throws Exception{
-Path directory = Paths.get(SecurityUtils.getSecurityDir());
-Files.walkFileTree(directory, new SimpleFileVisitor() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
- throws IOException {
-Files.delete(file);
-return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc)
- throws IOException {
-Files.delete(dir);
-return FileVisitResult.CONTINUE;
- }
-
-});
+FileUtils.deleteDirectory(new File(SecurityUtils.getSecurityDir()));
+//Path directory = Paths.get(SecurityUtils.getSecurityDir());
+//Files.walkFileTree(directory, new SimpleFileVisitor() {
+// @Override
+// public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+// throws IOException {
+//Files.delete(file);
+//return FileVisitResult.CONTINUE;
+// }
+//
+// @Override
+// public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+// throws IOException {
+//Files.delete(dir);
+//return FileVisitResult.CONTINUE;
+// }
+//
+//});
}
}
[49/50] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be1174cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be1174cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be1174cb Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: be1174cb310ea86e4fecdc573615c06ae727dbfb Parents: 6cac8a5 cbe2dd1 Author: Steve Loughran Authored: Tue Aug 26 18:31:23 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 26 18:31:23 2014 +0100 -- app-packages/accumulo/README.txt| 28 ++-- app-packages/accumulo/pom.xml | 5 +++ .../server/appmaster/SliderAppMaster.java | 25 +-- .../appmaster/monkey/ChaosMonkeyService.java| 46 ++-- .../model/monkey/TestMockMonkey.groovy | 41 ++--- 5 files changed, 120 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be1174cb/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[43/50] git commit: SLIDER-149 catching up with changes in YARN-913 branch
SLIDER-149 catching up with changes in YARN-913 branch
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6cac8a5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6cac8a5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6cac8a5b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6cac8a5ba9c0cb5d0e24c170250070482e85011b
Parents: 8a48392
Author: Steve Loughran
Authored: Tue Aug 26 17:35:08 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 26 17:35:08 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 40 +---
.../utility/AbstractSliderLaunchedService.java | 14 ---
2 files changed, 35 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6cac8a5b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 78aa923..ebf2133 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,6 +19,7 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -1168,11 +1169,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
DEFAULT_REGISTRY_PATH);
-addMandatoryConfOptionToCLI(commandLine, config,
RegistryConstants.REGISTRY_ZK_QUORUM);
+addMandatoryConfOptionToCLI(commandLine, config,
RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
-define(commandLine, RegistryConstants.REGISTRY_ZK_QUORUM,
-getRegistryOperations().getCurrentZookeeperQuorum());
-
if (clusterSecure) {
// if the cluster is secure, make sure that
@@ -1291,12 +1289,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
Configuration conf,
String key) {
String val = conf.get(key);
-if (val != null) {
- define(cmdLine, key, val);
- return true;
-} else {
- return false;
-}
+return defineIfSet(cmdLine, key, val);
}
private String addConfOptionToCLI(CommandLineBuilder cmdLine,
@@ -1309,15 +1302,34 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
/**
- * Add a define command to the CLI
- * @param cmdLine
- * @param key
- * @param val
+ * Add a -D key=val command to the CLI
+ * @param cmdLine command line
+ * @param key key
+ * @param val value
*/
private void define(CommandLineBuilder cmdLine, String key, String val) {
+Preconditions.checkArgument(key != null, "null key");
+Preconditions.checkArgument(val != null, "null value");
cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
}
+ /**
+ * Add a -D key=val command to the CLI if val
+ * is not null
+ * @param cmdLine command line
+ * @param key key
+ * @param val value
+ */
+ private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String
val) {
+Preconditions.checkArgument(key != null, "null key");
+if (val != null) {
+ define(cmdLine, key, val);
+ return true;
+} else {
+ return false;
+}
+ }
+
private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
Configuration conf,
String key) throws BadConfigException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6cac8a5b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 81ddfc2..3dafcd2 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -18,7 +18,6 @@
package org.apache.slider.server.services.utility;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hado
[30/50] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java slider-core/src/main/java/org/apache/slider/providers/ProviderService.java slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/434a964b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/434a964b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/434a964b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 434a964b65fad17e8155b5f76f579ff675d32394 Parents: 5de46b8 7b7d29e Author: Steve Loughran Authored: Mon Aug 25 14:37:26 2014 +0100 Committer: Steve Loughran Committed: Mon Aug 25 14:37:26 2014 +0100 -- app-packages/accumulo/pom.xml | 2 +- .../command-logger/application-pkg/pom.xml | 2 +- .../command-logger/slider-pkg/appConfig.json| 2 - .../command-logger/slider-pkg/metainfo.xml | 8 + app-packages/command-logger/slider-pkg/pom.xml | 2 +- app-packages/hbase/appConfig.json | 38 + app-packages/hbase/configuration/global.xml | 160 --- app-packages/hbase/configuration/hbase-env.xml | 117 ++ .../hbase/configuration/hbase-log4j.xml | 1 + app-packages/hbase/configuration/hbase-site.xml | 19 ++- app-packages/hbase/get-hbase-site.sh| 15 ++ app-packages/hbase/metainfo.xml | 24 +++ app-packages/hbase/package/scripts/hbase.py | 26 +-- app-packages/hbase/package/scripts/params.py| 26 ++- .../hbase/package/templates/hbase-env.sh.j2 | 81 -- app-packages/storm/appConfig.json | 88 +- app-packages/storm/configuration/global.xml | 39 - app-packages/storm/configuration/storm-env.xml | 40 + app-packages/storm/configuration/storm-site.xml | 15 +- app-packages/storm/metainfo.xml | 14 ++ app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/storm.py | 7 +- .../storm/package/scripts/yaml_config.py| 7 +- pom.xml | 4 +- slider-agent/pom.xml| 2 +- .../src/main/python/agent/Controller.py | 2 +- slider-agent/src/main/python/agent/Registry.py | 12 +- slider-agent/src/main/python/agent/main.py | 7 +- .../core/providers/__init__.py | 19 ++- .../core/providers/system.py| 8 + .../core/resources/system.py| 1 + .../python/resource_management/TestPackage.py | 32 slider-assembly/pom.xml | 2 +- slider-assembly/src/conf/slider-client.xml | 32 ++-- slider-assembly/src/main/scripts/slider | 3 +- slider-assembly/src/main/scripts/slider.py | 21 ++- slider-core/pom.xml | 2 +- .../java/org/apache/slider/api/StatusKeys.java | 7 +- .../registry/info/CustomRegistryConstants.java | 9 +- .../providers/AbstractProviderService.java | 15 +- .../slider/providers/ProviderService.java | 15 +- .../providers/agent/AgentProviderService.java | 160 ++- .../slider/providers/agent/AgentUtils.java | 31 +++- .../agent/application/metadata/Application.java | 11 +- .../agent/application/metadata/Component.java | 6 + .../agent/application/metadata/ConfigFile.java | 53 ++ .../metadata/ConfigurationDependencies.java | 39 - .../application/metadata/DefaultConfig.java | 39 + .../metadata/DefaultConfigParser.java | 54 +++ .../application/metadata/MetainfoParser.java| 9 +- .../application/metadata/PropertyInfo.java | 54 +++ .../slideram/SliderAMProviderService.java | 28 ++-- .../server/appmaster/SliderAppMaster.java | 49 +++--- .../server/appmaster/web/AgentService.java | 28 +--- .../appmaster/web/rest/agent/AgentWebApp.java | 16 +- .../server/services/security/SecurityUtils.java | 60 --- .../LaunchedWorkflowCompositeService.java | 4 + .../services/workflow/ClosingService.java | 3 + .../test_command_log/appConfig.json | 2 - .../test_command_log/a
[27/50] git commit: SLIDER-353 killJavaProcesses doesn't work on windows
SLIDER-353 killJavaProcesses doesn't work on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1d62a81d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1d62a81d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1d62a81d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1d62a81ddda40ebd6efc9159cf06c912817d2d55
Parents: 73389fd
Author: Steve Loughran
Authored: Mon Aug 25 12:39:01 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 25 12:39:01 2014 +0100
--
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1d62a81d/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 0a1354b..8057356 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -745,7 +745,7 @@ class SliderTestUtils extends Assert {
*/
public void killJavaProcesses(String grepString, int signal) {
-assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows
support ")
+//assume(!Shell.WINDOWS, "failed to kill -$signal $grepString - no windows
support ")
GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
log.info("Bash command = $bashCommand" )
[33/50] git commit: SLIDER-249 in sync with current YARN-913
SLIDER-249 in sync with current YARN-913
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/428f3a78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/428f3a78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/428f3a78
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 428f3a78fb418b96b9071ef96fa5365c21fe92be
Parents: 0c05abe
Author: Steve Loughran
Authored: Mon Aug 25 19:31:09 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 25 19:31:09 2014 +0100
--
.../providers/agent/AgentProviderService.java | 5 ++-
.../server/appmaster/SliderAppMaster.java | 34 +++-
.../actions/RegisterComponentInstance.java | 9 --
.../YarnRegistryViewForProviders.java | 6
4 files changed, 40 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/428f3a78/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f78d468..bd83d28 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -95,7 +95,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -583,8 +582,8 @@ public class AgentProviderService extends
AbstractProviderService implements
// and update registration entries
if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
- TimeUnit.MILLISECONDS));
+ queueAccess.put(new RegisterComponentInstance(instance.getId(),
+ roleName, 0, TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/428f3a78/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 835ef06..f28bd8f 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -365,6 +365,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
private final QueueService actionQueues = new QueueService();
private String agentOpsUrl;
private String agentStatusUrl;
+ private YarnRegistryViewForProviders yarnRegistryView;
/**
* Service Constructor
@@ -895,11 +896,10 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
setupInitialRegistryPaths();
-YarnRegistryViewForProviders yarnRegistryView =
-new YarnRegistryViewForProviders(
-registryOperations, service_user_name,
-SliderKeys.APP_TYPE,
-instanceName);
+yarnRegistryView = new YarnRegistryViewForProviders(
+registryOperations, service_user_name,
+SliderKeys.APP_TYPE,
+instanceName);
providerService.bindToYarnRegistry(yarnRegistryView);
sliderAMProvider.bindToYarnRegistry(yarnRegistryView);
@@ -978,15 +978,26 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
* Handler for {@link RegisterComponentInstance action}
* Register/re-register a component (that is already in the app state
* @param id the component
+ * @param description
*/
- public boolean registerComponent(ContainerId id) {
+ public boolean registerComponent(ContainerId id, String description) throws
+ IOException {
RoleInstance instance = appState.getOwnedContainer(id);
if (instance == null) {
return false;
}
// this is where component registrations will go
log.info("Registering component {}", id);
-
+String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
+ServiceRecord container = new ServiceRecord(
+cid,
+description);
+try {
+ yarnRegistryView.putComponent(cid, container, true);
+} catch (IOException e) {
+ log.warn("Failed to register container {}/{}: {}",
+ id, description, e, e);
+}
return true;
[32/50] git commit: SLIDER-358 added a clear log message and stack trace for openssl failures
SLIDER-358 added a clear log message and stack trace for openssl failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc9766be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc9766be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc9766be
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bc9766bea369e6083cb4007c7bb3d075a0a2f66f
Parents: 7b7d29e
Author: Jon Maron
Authored: Mon Aug 25 14:05:27 2014 -0400
Committer: Jon Maron
Committed: Mon Aug 25 14:05:27 2014 -0400
--
.../services/security/CertificateManager.java | 54
1 file changed, 32 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9766be/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 3771208..8f679e1 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -19,10 +19,9 @@ package org.apache.slider.server.services.security;
import com.google.inject.Singleton;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,7 +91,8 @@ public class CertificateManager {
*
* @return command execution exit code
*/
- private int runCommand(String command) {
+ private int runCommand(String command) throws SliderException {
+int exitCode = -1;
String line = null;
Process process = null;
BufferedReader br= null;
@@ -108,7 +108,10 @@ public class CertificateManager {
try {
process.waitFor();
SecurityUtils.logOpenSslExitCode(command, process.exitValue());
-return process.exitValue(); //command is executed
+exitCode = process.exitValue();
+if (exitCode != 0) {
+ throw new SliderException(exitCode, "Error running command {}",
command);
+}
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -124,11 +127,11 @@ public class CertificateManager {
}
}
-return -1;//some exception occurred
+return exitCode;//some exception occurred
}
- private void generateServerCertificate() {
+ private void generateServerCertificate(){
LOG.info("Generation of server certificate");
String srvrKstrDir = SecurityUtils.getSecurityDir();
@@ -141,17 +144,21 @@ public class CertificateManager {
Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName};
-String command = MessageFormat.format(GEN_SRVR_KEY,scriptArgs);
-runCommand(command);
+try {
+ String command = MessageFormat.format(GEN_SRVR_KEY,scriptArgs);
+ runCommand(command);
-command = MessageFormat.format(GEN_SRVR_REQ,scriptArgs);
-runCommand(command);
+ command = MessageFormat.format(GEN_SRVR_REQ,scriptArgs);
+ runCommand(command);
-command = MessageFormat.format(SIGN_SRVR_CRT,scriptArgs);
-runCommand(command);
+ command = MessageFormat.format(SIGN_SRVR_CRT,scriptArgs);
+ runCommand(command);
-command = MessageFormat.format(EXPRT_KSTR,scriptArgs);
-runCommand(command);
+ command = MessageFormat.format(EXPRT_KSTR,scriptArgs);
+ runCommand(command);
+} catch (SliderException e) {
+ LOG.error("Error generating the server certificate", e);
+}
}
@@ -205,11 +212,14 @@ public class CertificateManager {
//Revoke previous agent certificate if exists
File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+String command = null;
if (agentCrtFile.exists()) {
LOG.info("Revoking of " + agentHostname + " certificate.");
- String command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
- int commandExitCode = runCommand(command);
- if (commandExitCode != 0) {
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ try {
+runCommand(command);
+ } catch (SliderException e) {
+int commandExitCode = e.getExitCode();
response.setResult(SignCertResponse.ERROR_STATUS);
[24/50] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41599dc1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41599dc1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41599dc1 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 41599dc14cd259e755bc01c0d9febc6bb4d8fe71 Parents: 3736b3a 6afa8f9 Author: Sumit Mohanty Authored: Sat Aug 23 19:55:57 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 23 19:55:57 2014 -0700 -- slider-assembly/src/main/scripts/slider| 3 ++- slider-assembly/src/main/scripts/slider.py | 21 - 2 files changed, 22 insertions(+), 2 deletions(-) --
[40/50] git commit: SLIDER-361 removed use of StandardCharsets
SLIDER-361 removed use of StandardCharsets
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/61a5eac0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/61a5eac0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/61a5eac0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 61a5eac0c4d22eb51b19f53a0e509cd8393c873d
Parents: 4612ccb
Author: Billie Rinaldi
Authored: Tue Aug 26 04:01:21 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Aug 26 04:01:21 2014 -0700
--
.../java/org/apache/slider/accumulo/CustomAuthenticator.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/61a5eac0/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
--
diff --git
a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
index 322c339..0f50838 100644
---
a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
+++
b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
@@ -29,7 +29,7 @@ import
org.apache.accumulo.server.security.handler.PermissionHandler;
import org.apache.accumulo.server.security.handler.ZKAuthenticator;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.nio.charset.Charset;
import java.util.Set;
public final class CustomAuthenticator implements Authenticator {
@@ -68,7 +68,7 @@ public final class CustomAuthenticator implements
Authenticator {
ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile, ioe);
}
zkAuthenticator.initializeSecurity(credentials, principal,
-pass.getBytes(StandardCharsets.UTF_8));
+pass.getBytes(Charset.forName("UTF-8")));
}
@Override
[01/50] git commit: SLIDER-348. Use command timeout specified in the metainfo
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry b0025d60d ->
a0528889f
SLIDER-348. Use command timeout specified in the metainfo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d66d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d66d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d66d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d66dfd0a3da9aa0f34b354c530a323081ff2
Parents: 111e03f
Author: Sumit Mohanty
Authored: Thu Aug 21 23:30:17 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 21 23:30:17 2014 -0700
--
.../providers/agent/AgentProviderService.java | 44 +---
.../agent/application/metadata/Component.java | 6 +++
.../agent/TestAgentProviderService.java | 32 ++
3 files changed, 59 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d66d/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index ec5425a..562ca1f 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -56,6 +56,7 @@ import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.ConfigFile;
@@ -454,13 +455,20 @@ public class AgentProviderService extends
AbstractProviderService implements
String containerId = getContainerId(label);
StateAccessForProviders accessor = getAmState();
-String scriptPath = getScriptPathFromMetainfo(roleName);
+CommandScript cmdScript = getScriptPathFromMetainfo(roleName);
-if (scriptPath == null) {
+if (cmdScript == null || cmdScript.getScript() == null) {
log.error("role.script is unavailable for " + roleName + ". Commands
will not be sent.");
return response;
}
+String scriptPath = cmdScript.getScript();
+long timeout = cmdScript.getTimeout();
+
+if(timeout == 0L) {
+ timeout = 600L;
+}
+
if (!getComponentStatuses().containsKey(label)) {
return response;
}
@@ -502,14 +510,14 @@ public class AgentProviderService extends
AbstractProviderService implements
if (Command.NOP != command) {
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- addInstallCommand(roleName, containerId, response, scriptPath);
+ addInstallCommand(roleName, containerId, response, scriptPath,
timeout);
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
boolean canExecute = commandOrder.canExecute(roleName, command,
getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
-addStartCommand(roleName, containerId, response, scriptPath,
isMarkedAutoRestart(roleName));
+addStartCommand(roleName, containerId, response, scriptPath,
timeout, isMarkedAutoRestart(roleName));
componentStatus.commandIssued(command);
} else {
log.info("Start of {} on {} delayed as dependencies have not
started.", roleName, containerId);
@@ -1037,10 +1045,10 @@ public class AgentProviderService extends
AbstractProviderService implements
*
* @return
*/
- protected String getScriptPathFromMetainfo(String roleName) {
+ protected CommandScript getScriptPathFromMetainfo(String roleName) {
Component component = getApplicationComponent(roleName);
if (component != null) {
- return component.getCommandScript().getScript();
+ return component.getCommandScript();
}
return null;
}
@@ -1134,7 +1142,11 @@ public class AgentProviderService extends
AbstractProviderService implements
* @throws SliderException
*/
@VisibleForTesting
- protected void addInstallCommand(Stri
[22/50] git commit: SLIDER-254 added ASF snapshots repository to pull in snapshot versions of accumulo and hadoop
SLIDER-254 added ASF snapshots repository to pull in snapshot versions of
accumulo and hadoop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c89b264e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c89b264e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c89b264e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c89b264e9cd38698eb6a9dec7d979c873e90baa2
Parents: 3178498
Author: Billie Rinaldi
Authored: Fri Aug 22 18:57:15 2014 -0700
Committer: Billie Rinaldi
Committed: Fri Aug 22 18:57:15 2014 -0700
--
app-packages/accumulo/pom.xml | 18 ++
pom.xml | 10 ++
2 files changed, 28 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c89b264e/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index d3f83a7..c2cbe1a 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -146,6 +146,24 @@
${accumulo.version}
bin
tar.gz
+
+
+ org.apache.accumulo
+ accumulo-fate
+
+
+ org.apache.accumulo
+ accumulo-gc
+
+
+ org.apache.accumulo
+ accumulo-master
+
+
+ org.apache.accumulo
+ accumulo-minicluster
+
+
org.apache.accumulo
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c89b264e/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ab6e9a6..dffa493 100644
--- a/pom.xml
+++ b/pom.xml
@@ -207,6 +207,16 @@
ASF Staging
https://repository.apache.org/content/groups/staging/
+
+ ASF Snapshots
+ https://repository.apache.org/content/repositories/snapshots/
+
+true
+
+
+false
+
+
[48/50] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cbe2dd1a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cbe2dd1a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cbe2dd1a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: cbe2dd1a69816680abd282f506696e21f6e3932c Parents: fb22114 d497715 Author: Steve Loughran Authored: Tue Aug 26 18:30:46 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 26 18:30:46 2014 +0100 -- app-packages/accumulo/pom.xml | 5 + 1 file changed, 5 insertions(+) --
[20/50] git commit: SLIDER-352 making one of the tests skipped if !Windows
SLIDER-352 making one of the tests skipped if !Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/31d70a34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/31d70a34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/31d70a34
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 31d70a348c5c96aee92f26ea767f19ddc56fc710
Parents: aa21ad6
Author: Steve Loughran
Authored: Fri Aug 22 21:37:39 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 22 21:37:39 2014 +0100
--
.../org/apache/slider/common/tools/TestWindowsSupport.groovy | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31d70a34/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index e6d2225..d11aa55 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -93,6 +93,8 @@ class TestWindowsSupport extends SliderTestUtils {
@Test
public void testSliderFS() throws Throwable {
+assume(Shell.WINDOWS, "not windows")
+
SliderFileSystem sfs = new SliderFileSystem(new Configuration())
try {
def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile)
[50/50] git commit: SLIDER-149 complete client switch to new registry code ... old code stil present but unused. Tests working
SLIDER-149 complete client switch to new registry code ... old code stil
present but unused. Tests working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a0528889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a0528889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a0528889
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a0528889fc47977794efdc2354597c7289e382b8
Parents: be1174c
Author: Steve Loughran
Authored: Wed Aug 27 20:57:31 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 27 20:57:31 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 172 -
.../registry/info/CommonRegistryConstants.java | 25 --
.../registry/info/CustomRegistryConstants.java | 7 +
.../registry/retrieve/RegistryRetriever.java| 62 +--
.../slideram/SliderAMProviderService.java | 17 +-
.../server/appmaster/SliderAppMaster.java | 37 +-
.../utility/AbstractSliderLaunchedService.java | 15 +-
.../YarnRegistryViewForProviders.java | 7 +-
.../standalone/TestStandaloneRegistryAM.groovy | 367 --
.../agent/standalone/TestYarnRegistryAM.groovy | 376 +++
.../apache/slider/test/MicroZKCluster.groovy| 29 +-
.../apache/slider/test/SliderTestUtils.groovy | 10 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../test/YarnZKMiniClusterTestBase.groovy | 21 +-
14 files changed, 671 insertions(+), 481 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a0528889/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ebf2133..0f7fe5a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -20,8 +20,10 @@ package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,8 +40,13 @@ import
org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.client.binding.BindingUtils;
+import org.apache.hadoop.yarn.registry.client.binding.RecordOperations;
import org.apache.hadoop.yarn.registry.client.binding.ZKPathDumper;
import
org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+import org.apache.hadoop.yarn.registry.client.types.Endpoint;
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
@@ -2290,14 +2297,14 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
registryArgs.validate();
try {
if (registryArgs.list) {
-actionRegistryList(registryArgs);
+actionRegistryListYarn(registryArgs);
} else if (registryArgs.listConf) {
// list the configurations
-actionRegistryListConfigs(registryArgs);
+actionRegistryListConfigsYarn(registryArgs);
} else if (SliderUtils.isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
-actionRegistryGetConfig(registryArgs);
+actionRegistryGetConfigYarn(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
} else {
// it's an unknown command
@@ -2305,9 +2312,14 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
"Bad command arguments for " + ACTION_REGISTRY + " " +
registryArgs);
}
+// JDK7
} catch (FileNotFoundException e) {
log.info("{}", e.toString());
- log.debug("{}",e, e);
+ log.debug("{}", e, e);
+ return EXIT_NOT_FOUND;
+} catch (PathNotFoundException e) {
+ log.info("{}", e.toString());
+ log.debug("{}", e, e);
re
[19/50] git commit: Merge branch 'feature/SLIDER-350-slider-core_tests_failing_on_windows' into develop
Merge branch 'feature/SLIDER-350-slider-core_tests_failing_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aa21ad64 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aa21ad64 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aa21ad64 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: aa21ad64d201651c154114e6fd187ca156807552 Parents: 4e8c6ff 87ff4d8 Author: Steve Loughran Authored: Fri Aug 22 18:54:42 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 22 18:54:42 2014 +0100 -- .../slider/providers/agent/AgentUtils.java | 11 +- .../server/appmaster/SliderAppMaster.java | 2 +- .../server/appmaster/web/AgentService.java | 28 ++--- .../appmaster/web/rest/agent/AgentWebApp.java | 16 ++- .../LaunchedWorkflowCompositeService.java | 4 + .../services/workflow/ClosingService.java | 3 + .../agent/AgentMiniClusterTestBase.groovy | 9 +- .../common/tools/TestWindowsSupport.groovy | 104 +++ .../publisher/TestPublisherRestResources.groovy | 11 +- .../slider/test/YarnMiniClusterTestBase.groovy | 4 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../workflow/TestWorkflowClosingService.java| 4 +- .../providers/accumulo/AccumuloTestBase.groovy | 7 +- .../minicluster/HBaseMiniClusterTestBase.groovy | 9 +- 14 files changed, 173 insertions(+), 42 deletions(-) --
[07/12] git commit: Merge branch 'feature/hbase_win_package' into develop
Merge branch 'feature/hbase_win_package' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/15dc1ecf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/15dc1ecf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/15dc1ecf Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 15dc1ecfb29076c59a6146d3c23e795477e2f6fd Parents: cbe2dd1 b3777fd Author: Sumit Mohanty Authored: Tue Aug 26 18:14:07 2014 -0700 Committer: Sumit Mohanty Committed: Tue Aug 26 18:14:07 2014 -0700 -- app-packages/app-pkg-template/metainfo.xml | 8 +- app-packages/hbase-win/README.txt | 34 ++ app-packages/hbase-win/appConfig.json | 44 +++ .../hbase-win/configuration/hbase-env.xml | 54 +++ .../hbase-win/configuration/hbase-log4j.xml | 143 +++ .../hbase-win/configuration/hbase-policy.xml| 53 +++ .../hbase-win/configuration/hbase-site.xml | 370 +++ app-packages/hbase-win/jmx_metrics.json | 56 +++ app-packages/hbase-win/metainfo.xml | 129 +++ ...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 + .../hbase-win/package/scripts/__init__.py | 19 + .../hbase-win/package/scripts/functions.py | 40 ++ app-packages/hbase-win/package/scripts/hbase.py | 66 .../hbase-win/package/scripts/hbase_master.py | 63 .../package/scripts/hbase_regionserver.py | 63 .../hbase-win/package/scripts/hbase_service.py | 48 +++ .../hbase-win/package/scripts/params.py | 81 .../hbase-win/package/scripts/status_params.py | 26 ++ .../package/templates/hbase_client_jaas.conf.j2 | 22 ++ .../package/templates/hbase_master_jaas.conf.j2 | 25 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 25 ++ app-packages/hbase-win/resources.json | 21 ++ app-packages/hbase/appConfig.json | 3 +- app-packages/hbase/package/scripts/hbase.py | 2 + app-packages/memcached-win/appConfig.json | 6 +- app-packages/memcached-win/metainfo.xml | 8 +- app-packages/memcached/metainfo.xml | 8 +- slider-agent/src/main/python/agent/Constants.py | 1 + .../src/main/python/agent/Controller.py | 1 - slider-agent/src/main/python/agent/Registry.py | 8 +- slider-agent/src/main/python/agent/main.py | 59 +-- slider-agent/src/main/python/kazoo/client.py| 6 +- .../src/main/python/kazoo/handlers/threading.py | 6 +- .../src/main/python/kazoo/handlers/utils.py | 87 - .../main/python/kazoo/protocol/connection.py| 16 +- .../src/main/python/kazoo/tests/test_client.py | 6 +- .../main/python/kazoo/tests/test_connection.py | 55 +-- .../core/providers/__init__.py | 14 +- .../core/providers/windows/system.py| 74 +++- .../core/providers/windows/tarball.py | 46 +++ .../libraries/functions/check_process_status.py | 44 ++- .../libraries/script/script.py | 2 +- .../providers/agent/AgentProviderService.java | 5 + 43 files changed, 1739 insertions(+), 125 deletions(-) --
[06/12] git commit: SLIDER-359. Create hbase package for Windows
SLIDER-359. Create hbase package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3777fde
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3777fde
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3777fde
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b3777fde109cf2e9c9cf22d674d571e7ce845918
Parents: 829821a
Author: Sumit Mohanty
Authored: Tue Aug 26 18:13:29 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 26 18:13:29 2014 -0700
--
app-packages/hbase-win/README.txt | 1 +
app-packages/hbase-win/appConfig.json | 6 +-
.../hbase-win/configuration/hbase-env.xml | 67 +---
app-packages/hbase-win/metainfo.xml | 10 +++
app-packages/hbase-win/package/scripts/hbase.py | 15 +---
.../hbase-win/package/scripts/hbase_master.py | 8 +-
.../package/scripts/hbase_regionserver.py | 8 +-
.../hbase-win/package/scripts/hbase_service.py | 53 ++---
.../hbase-win/package/scripts/params.py | 82 +++-
app-packages/hbase/appConfig.json | 3 +-
app-packages/hbase/package/scripts/hbase.py | 2 +
.../core/providers/__init__.py | 1 +
.../core/providers/windows/system.py| 29 +++
.../libraries/functions/check_process_status.py | 44 ---
14 files changed, 127 insertions(+), 202 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/README.txt
--
diff --git a/app-packages/hbase-win/README.txt
b/app-packages/hbase-win/README.txt
index c122d90..625bce6 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -22,6 +22,7 @@ Download the tarball for HBase:
Copy the hbase tarball to package/files
cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files
+ rm -rf package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
Edit appConfig.json/metainfo.xml and replace all occurances of
0.98.0.2.1.5.0-2047-hadoop2 with the correct value.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/appConfig.json
--
diff --git a/app-packages/hbase-win/appConfig.json
b/app-packages/hbase-win/appConfig.json
index 52f3aa3..8261f8c 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -21,15 +21,15 @@
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+"site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+"site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
"site.hbase-env.hbase_master_heapsize": "1024m",
"site.hbase-env.hbase_regionserver_heapsize": "1024m",
-"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
-"site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
+"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
"site.hbase-site.hbase.superuser": "yarn",
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
-"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/configuration/hbase-env.xml
--
diff --git a/app-packages/hbase-win/configuration/hbase-env.xml
b/app-packages/hbase-win/configuration/hbase-env.xml
index 554c3c5..fa5686f 100644
--- a/app-packages/hbase-win/configuration/hbase-env.xml
+++ b/app-packages/hbase-win/configuration/hbase-env.xml
@@ -45,72 +45,9 @@
content
-This is the jinja template for hbase-env.sh file
+This is the jinja template for start command
-# Set environment variables here.
-
-# The java implementation to use. Java 1.6 required.
-export JAVA_HOME={{java64_home}}
-
-# HBase Configuration directory
-export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}}
-
-# Extra Java CLASSPATH elements. Optional.
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-# export HBASE_HEAPSIZE=1000
-
-# Extra Java runtime options.
-# Below are what we set by default. May only work with SUN JVM.
-# For more on why as well as other pos
[05/12] git commit: SLIDER-359. Create hbase package for Windows
SLIDER-359. Create hbase package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/829821a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/829821a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/829821a0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 829821a000cd175611ee346d1f48dae462469ea4
Parents: 7bc331d
Author: Sumit Mohanty
Authored: Mon Aug 25 13:54:40 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Aug 25 13:54:40 2014 -0700
--
app-packages/hbase-win/README.txt | 33 ++
app-packages/hbase-win/appConfig.json | 44 +++
.../hbase-win/configuration/hbase-env.xml | 117 ++
.../hbase-win/configuration/hbase-log4j.xml | 143 +++
.../hbase-win/configuration/hbase-policy.xml| 53 +++
.../hbase-win/configuration/hbase-site.xml | 370 +++
app-packages/hbase-win/jmx_metrics.json | 56 +++
app-packages/hbase-win/metainfo.xml | 119 ++
...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 +
.../hbase-win/package/scripts/__init__.py | 19 +
.../hbase-win/package/scripts/functions.py | 40 ++
app-packages/hbase-win/package/scripts/hbase.py | 79
.../hbase-win/package/scripts/hbase_master.py | 63
.../package/scripts/hbase_regionserver.py | 63
.../hbase-win/package/scripts/hbase_service.py | 51 +++
.../hbase-win/package/scripts/params.py | 107 ++
.../hbase-win/package/scripts/status_params.py | 26 ++
.../package/templates/hbase_client_jaas.conf.j2 | 22 ++
.../package/templates/hbase_master_jaas.conf.j2 | 25 ++
.../templates/hbase_regionserver_jaas.conf.j2 | 25 ++
app-packages/hbase-win/resources.json | 21 ++
.../core/providers/windows/system.py| 79 +++-
.../providers/agent/AgentProviderService.java | 2 +
23 files changed, 1560 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/README.txt
--
diff --git a/app-packages/hbase-win/README.txt
b/app-packages/hbase-win/README.txt
new file mode 100644
index 000..c122d90
--- /dev/null
+++ b/app-packages/hbase-win/README.txt
@@ -0,0 +1,33 @@
+
+
+Create Slider App Package for HBase on Windows
+
+Download the tarball for HBase:
+ e.g. path to tarball ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip
+
+Copy the hbase tarball to package/files
+ cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files
+
+Edit appConfig.json/metainfo.xml and replace all occurances of
+0.98.0.2.1.5.0-2047-hadoop2 with the correct value.
+
+Create a zip package at the root of the package (/app-packages/hbase-win/)
+ zip -r hbase-win-v098.zip .
+
+Verify the content using
+ zip -Tv hbase-win-v098.zip
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/829821a0/app-packages/hbase-win/appConfig.json
--
diff --git a/app-packages/hbase-win/appConfig.json
b/app-packages/hbase-win/appConfig.json
new file mode 100644
index 000..52f3aa3
--- /dev/null
+++ b/app-packages/hbase-win/appConfig.json
@@ -0,0 +1,44 @@
+{
+ "schema": "http://example.org/specification/v2.0.0";,
+ "metadata": {
+ },
+ "global": {
+"application.def": "${app.package.name}.zip",
+"create.default.zookeeper.node": "true",
+"java_home": "/usr/jdk64/jdk1.7.0_45",
+
+"site.global.app_user": "hadoop",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+
+"site.global.hbase_instance_name": "instancename",
+"site.global.hbase_root_password": "secret",
+"site.global.user_group": "hadoop",
+"site.global.security_enabled": "false",
+"site.global.monitor_protocol": "http",
+"site.global.ganglia_server_host": "${NN_HOST}",
+"site.global.ganglia_server_port": "8667",
+"site.global.ganglia_server_id": "Application1",
+"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+
+"site.hbase-env.hbase_master_heapsize": "1024m",
+"site.hbase-env.hbase_regionserver_heapsize": "1024m",
+
+"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
+"site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
+"site.hbase-site.hbase.superuser": "yarn",
+"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}
[10/12] git commit: SLIDER-366. App package clean up.
SLIDER-366. App package clean up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b126e7cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b126e7cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b126e7cb
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b126e7cbeab1051fac0d89dae23a69f7f999e56a
Parents: 38e6d34
Author: Sumit Mohanty
Authored: Wed Aug 27 12:35:34 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Aug 27 12:35:34 2014 -0700
--
app-packages/app-pkg-template/appConfig.json | 2 +-
app-packages/memcached-win/appConfig.json| 2 +-
app-packages/memcached/appConfig.json| 2 +-
app-packages/storm/appConfig.json| 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b126e7cb/app-packages/app-pkg-template/appConfig.json
--
diff --git a/app-packages/app-pkg-template/appConfig.json
b/app-packages/app-pkg-template/appConfig.json
index a6f61f9..3908030 100644
--- a/app-packages/app-pkg-template/appConfig.json
+++ b/app-packages/app-pkg-template/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "package/myapp-1.0.0.zip",
+"application.def": "myapp-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b126e7cb/app-packages/memcached-win/appConfig.json
--
diff --git a/app-packages/memcached-win/appConfig.json
b/app-packages/memcached-win/appConfig.json
index 471a974..fa4050b 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "package/jmemcached-1.0.0.zip",
+"application.def": "jmemcached-1.0.0.zip",
"java_home": "C:\\java",
"site.global.app_user": "hadoop",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b126e7cb/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index 5f32030..7a97f98 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "package/jmemcached-1.0.0.zip",
+"application.def": "jmemcached-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b126e7cb/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index 1322a01..b8454f9 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "package/storm_v091.zip",
+"application.def": "storm_v091.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
[03/12] git commit: Merge branch 'develop' into feature/hbase_win_package
Merge branch 'develop' into feature/hbase_win_package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6afba78 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6afba78 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6afba78 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: a6afba78b2f5787e9cdc3d9512dd831613482cf3 Parents: 448d598 41599dc Author: Sumit Mohanty Authored: Sat Aug 23 19:57:45 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 23 19:57:45 2014 -0700 -- .../command-logger/slider-pkg/appConfig.json| 2 - .../command-logger/slider-pkg/metainfo.xml | 8 + app-packages/hbase/appConfig.json | 38 + app-packages/hbase/configuration/global.xml | 160 --- app-packages/hbase/configuration/hbase-env.xml | 117 ++ .../hbase/configuration/hbase-log4j.xml | 1 + app-packages/hbase/configuration/hbase-site.xml | 19 ++- app-packages/hbase/metainfo.xml | 24 +++ app-packages/hbase/package/scripts/hbase.py | 26 +-- app-packages/hbase/package/scripts/params.py| 26 ++- .../hbase/package/templates/hbase-env.sh.j2 | 81 -- app-packages/storm/appConfig.json | 88 +- app-packages/storm/configuration/global.xml | 39 - app-packages/storm/configuration/storm-env.xml | 40 + app-packages/storm/configuration/storm-site.xml | 15 +- app-packages/storm/metainfo.xml | 14 ++ app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/storm.py | 7 +- .../storm/package/scripts/yaml_config.py| 7 +- pom.xml | 1 + .../core/providers/system.py| 8 + .../core/resources/system.py| 1 + slider-assembly/src/main/scripts/slider | 3 +- slider-assembly/src/main/scripts/slider.py | 21 ++- .../providers/agent/AgentProviderService.java | 119 +++--- .../slider/providers/agent/AgentUtils.java | 31 +++- .../agent/application/metadata/Application.java | 11 +- .../agent/application/metadata/Component.java | 6 + .../agent/application/metadata/ConfigFile.java | 53 ++ .../metadata/ConfigurationDependencies.java | 39 - .../application/metadata/DefaultConfig.java | 39 + .../metadata/DefaultConfigParser.java | 54 +++ .../application/metadata/MetainfoParser.java| 9 +- .../application/metadata/PropertyInfo.java | 54 +++ .../server/appmaster/SliderAppMaster.java | 2 +- .../server/appmaster/web/AgentService.java | 28 +--- .../appmaster/web/rest/agent/AgentWebApp.java | 16 +- .../LaunchedWorkflowCompositeService.java | 4 + .../services/workflow/ClosingService.java | 3 + .../test_command_log/appConfig.json | 2 - .../test_command_log/appConfig_fast_no_reg.json | 2 - .../test_command_log/appConfig_no_hb.json | 2 - .../agent/AgentMiniClusterTestBase.groovy | 9 +- .../common/tools/TestWindowsSupport.groovy | 106 .../publisher/TestPublisherRestResources.groovy | 11 +- .../slider/test/YarnMiniClusterTestBase.groovy | 4 +- .../agent/TestAgentProviderService.java | 101 ++-- .../application/metadata/TestConfigParser.java | 107 + .../metadata/TestMetainfoParser.java| 7 + .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../workflow/TestWorkflowClosingService.java| 4 +- .../slider/common/tools/test/metainfo.xml | 11 +- .../providers/accumulo/AccumuloTestBase.groovy | 7 +- .../minicluster/HBaseMiniClusterTestBase.groovy | 9 +- 54 files changed, 1016 insertions(+), 591 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6afba78/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java --
[08/12] git commit: Unit test fixes to handle ZK retry in main
Unit test fixes to handle ZK retry in main
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f1552505
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f1552505
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f1552505
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f1552505033106c8259fd2230424c6c3b0128cc2
Parents: 15dc1ec
Author: Sumit Mohanty
Authored: Tue Aug 26 21:36:41 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 26 21:36:41 2014 -0700
--
slider-agent/src/main/python/agent/Registry.py | 1 +
slider-agent/src/main/python/agent/main.py | 3 ++-
slider-agent/src/test/python/agent/TestMain.py | 19 ++-
3 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1552505/slider-agent/src/main/python/agent/Registry.py
--
diff --git a/slider-agent/src/main/python/agent/Registry.py
b/slider-agent/src/main/python/agent/Registry.py
index 9d4a387..453be10 100644
--- a/slider-agent/src/main/python/agent/Registry.py
+++ b/slider-agent/src/main/python/agent/Registry.py
@@ -30,6 +30,7 @@ class Registry:
self.zk_reg_path = zk_reg_path
def readAMHostPort(self):
+logger.debug("Trying to connect to ZK...")
amHost = ""
amSecuredPort = ""
amUnsecuredPort = ""
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1552505/slider-agent/src/main/python/agent/main.py
--
diff --git a/slider-agent/src/main/python/agent/main.py
b/slider-agent/src/main/python/agent/main.py
index f6b8666..a8fc4e1 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -271,11 +271,12 @@ def main():
# Wait until server is reachable and continue to query ZK
netutil = NetUtil()
retries = netutil.try_to_connect(server_url, 3, logger)
- zk_lookup_tries += 1
if retries < 3:
break;
pass
pass
+zk_lookup_tries += 1
+ pass
# Launch Controller communication
controller = Controller(agentConfig)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1552505/slider-agent/src/test/python/agent/TestMain.py
--
diff --git a/slider-agent/src/test/python/agent/TestMain.py
b/slider-agent/src/test/python/agent/TestMain.py
index bc68582..e73a05a 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -31,6 +31,7 @@ import ConfigParser
import os
import tempfile
from Controller import Controller
+from Registry import Registry
from optparse import OptionParser
logger = logging.getLogger()
@@ -46,7 +47,6 @@ class TestMain(unittest.TestCase):
# enable stdout
sys.stdout = sys.__stdout__
-
@patch("os._exit")
@patch("os.getpid")
@patch.object(ProcessHelper, "stopAgent")
@@ -218,7 +218,7 @@ class TestMain(unittest.TestCase):
ProcessHelper.pidfile = oldpid
os.remove(tmpoutfile)
-
+ @patch.object(Registry, "readAMHostPort")
@patch.object(main, "setup_logging")
@patch.object(main, "bind_signal_handlers")
@patch.object(main, "update_config_from_file")
@@ -236,17 +236,20 @@ class TestMain(unittest.TestCase):
update_log_level_mock, write_pid_mock,
perform_prestart_checks_mock,
update_config_from_file_mock,
-bind_signal_handlers_mock, setup_logging_mock):
+bind_signal_handlers_mock, setup_logging_mock,
+readAMHostPort_mock):
Controller_init_mock.return_value = None
isAlive_mock.return_value = False
options = MagicMock()
parse_args_mock.return_value = (options, MagicMock)
+readAMHostPort_mock.return_value = ("host1", 101, 100)
tmpdir = tempfile.gettempdir()
#testing call without command-line arguments
os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
os.environ["AGENT_LOG_ROOT"] = ",".join([os.path.join(tmpdir,
"log"),os.path.join(tmpdir, "log2")])
+try_to_connect_mock.return_value = 1
main.main()
self.assertTrue(setup_logging_mock.called)
@@ -256,7 +259,7 @@ class TestMain(unittest.TestCase):
self.assertTrue(write_pid_mock.called)
self.assertTrue(update_log_level_mock.called)
self.assertTrue(options.log_folder == os.path.join(tmpdir, "log"))
-try_to_connect_mock.assert_called_once_with(ANY, -1, ANY)
+
try_to_connect_mock.assert_called_once_with('https://host1:101/ws/v1/slider/agents/',
3, ANY)
self.assertTrue(start_mock.called)
c
[04/12] git commit: SLIDER-357. slider bash script cannot handle if java is in the path
SLIDER-357. slider bash script cannot handle if java is in the path
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7bc331d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7bc331d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7bc331d3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7bc331d3d37bb05ec77dbe1f0f8d0f8e7a2ec906
Parents: a6afba7
Author: Sumit Mohanty
Authored: Sat Aug 23 20:53:36 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Aug 23 20:53:36 2014 -0700
--
slider-assembly/src/main/scripts/slider | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7bc331d3/slider-assembly/src/main/scripts/slider
--
diff --git a/slider-assembly/src/main/scripts/slider
b/slider-assembly/src/main/scripts/slider
index 7172917..6cd55ad 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -71,5 +71,5 @@ echo ""
echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\"
${launcher} $@"
echo ""
echo ""
-type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin
-exec ${javabin}/java ${slider_jvm_opts} ${launcher} $@
+type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin/
+exec ${javabin}java ${slider_jvm_opts} ${launcher} $@
[01/12] git commit: SLIDER-354. Add support for zip packages containing application bits.
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry a0528889f ->
c692461b7
SLIDER-354. Add support for zip packages containing application bits.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fc534ca7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fc534ca7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fc534ca7
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: fc534ca7b0ecf607e9278bd352847a7e62db1ca6
Parents: 644dd38
Author: Sumit Mohanty
Authored: Fri Aug 22 17:39:37 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Aug 22 17:39:37 2014 -0700
--
app-packages/app-pkg-template/metainfo.xml | 8 ++--
app-packages/memcached-win/appConfig.json | 7 +--
app-packages/memcached-win/metainfo.xml | 12 ++---
app-packages/memcached/metainfo.xml | 8 ++--
.../core/providers/__init__.py | 13 +++---
.../core/providers/windows/tarball.py | 46
.../libraries/script/script.py | 2 +-
7 files changed, 70 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc534ca7/app-packages/app-pkg-template/metainfo.xml
--
diff --git a/app-packages/app-pkg-template/metainfo.xml
b/app-packages/app-pkg-template/metainfo.xml
index c6e1485..50c0fbd 100644
--- a/app-packages/app-pkg-template/metainfo.xml
+++ b/app-packages/app-pkg-template/metainfo.xml
@@ -28,12 +28,12 @@
MYAPP_COMPONENT
MASTER
-
-
+
+
host_port
${THIS_HOST}:${site.global.listen_port}
-
-
+
+
scripts/myapp_component.py
PYTHON
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc534ca7/app-packages/memcached-win/appConfig.json
--
diff --git a/app-packages/memcached-win/appConfig.json
b/app-packages/memcached-win/appConfig.json
index b76ecde..e39feb0 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -3,12 +3,11 @@
"metadata": {
},
"global": {
-"application.def": "/slider/jmemcached-1.0.0.zip",
+"application.def": "package/jmemcached-1.0.0.zip",
"java_home": "C:\\java",
"site.global.app_user": "hadoop",
-"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
-"site.global.pid_file": "${AGENT_WORK_ROOT}\\app\\run\\component.pid",
+"site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
"site.global.additional_cp":
"C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
@@ -19,8 +18,6 @@
"components": {
"slider-appmaster": {
"jvm.heapsize": "256M"
-},
-"MEMCACHED": {
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc534ca7/app-packages/memcached-win/metainfo.xml
--
diff --git a/app-packages/memcached-win/metainfo.xml
b/app-packages/memcached-win/metainfo.xml
index d056c0a..27a8be6 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -28,12 +28,12 @@
MEMCACHED
MASTER
-
-
+
+
host_port
${THIS_HOST}:${site.global.listen_port}
-
-
+
+
scripts/memcached.py
PYTHON
@@ -46,8 +46,8 @@
any
-folder
-files\\jmemcached-1.0.0
+zip
+files/jmemcached-1.0.0.zip
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc534ca7/app-packages/memcached/metainfo.xml
--
diff --git a/app-packages/memcached/metainfo.xml
b/app-packages/memcached/metainfo.xml
index 525816e..5801ad2 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -28,12 +28,12 @@
MEMCACHED
MASTER
-
-
+
+
host_port
${THIS_HOST}:${site.global.listen_port}
-
-
+
+
scripts/memcached.py
PYTHON
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc534ca7/slider-agent/src/main/python/resource_management/core/providers/__init__.py
--
[02/12] git commit: SLIDER-355. Agent is unable to connect to ZK on windows to retrieve AM host and port SLIDER-356. Slider agent should not default to any host or port value when it cannot communicat
SLIDER-355. Agent is unable to connect to ZK on windows to retrieve AM host and
port
SLIDER-356. Slider agent should not default to any host or port value when it
cannot communicate with ZK
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/448d598f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/448d598f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/448d598f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 448d598fccf75d2a340cdbbeb45d3913fb473050
Parents: fc534ca
Author: Sumit Mohanty
Authored: Sat Aug 23 19:50:16 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Aug 23 19:50:30 2014 -0700
--
app-packages/memcached-win/appConfig.json | 1 +
app-packages/memcached-win/metainfo.xml | 4 +-
slider-agent/src/main/python/agent/Constants.py | 1 +
.../src/main/python/agent/Controller.py | 1 -
slider-agent/src/main/python/agent/Registry.py | 8 +-
slider-agent/src/main/python/agent/main.py | 59 -
slider-agent/src/main/python/kazoo/client.py| 6 +-
.../src/main/python/kazoo/handlers/threading.py | 6 +-
.../src/main/python/kazoo/handlers/utils.py | 87 +---
.../main/python/kazoo/protocol/connection.py| 16 ++--
.../src/main/python/kazoo/tests/test_client.py | 6 +-
.../main/python/kazoo/tests/test_connection.py | 55 -
.../providers/agent/AgentProviderService.java | 3 +
13 files changed, 176 insertions(+), 77 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/448d598f/app-packages/memcached-win/appConfig.json
--
diff --git a/app-packages/memcached-win/appConfig.json
b/app-packages/memcached-win/appConfig.json
index e39feb0..471a974 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -8,6 +8,7 @@
"site.global.app_user": "hadoop",
"site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
+"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
"site.global.additional_cp":
"C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/448d598f/app-packages/memcached-win/metainfo.xml
--
diff --git a/app-packages/memcached-win/metainfo.xml
b/app-packages/memcached-win/metainfo.xml
index 27a8be6..c7e5881 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -46,8 +46,8 @@
any
-zip
-files/jmemcached-1.0.0.zip
+folder
+files\\jmemcached-1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/448d598f/slider-agent/src/main/python/agent/Constants.py
--
diff --git a/slider-agent/src/main/python/agent/Constants.py
b/slider-agent/src/main/python/agent/Constants.py
index 2975266..f120b94 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -33,3 +33,4 @@ DO_NOT_HEARTBEAT_AFTER_ = "DO_NOT_HEARTBEAT_AFTER_"
ZK_QUORUM="zk_quorum"
ZK_REG_PATH="zk_reg_path"
AUTO_GENERATED="auto_generated"
+MAX_AM_CONNECT_RETRIES = 10
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/448d598f/slider-agent/src/main/python/agent/Controller.py
--
diff --git a/slider-agent/src/main/python/agent/Controller.py
b/slider-agent/src/main/python/agent/Controller.py
index dfd0a09..a3fb90d 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -331,7 +331,6 @@ class Controller(threading.Thread):
zk_reg_path = self.config.get(AgentConfig.SERVER_SECTION,
Constants.ZK_REG_PATH)
registry = Registry(zk_quorum, zk_reg_path)
amHost, amUnsecuredPort, amSecuredPort = registry.readAMHostPort()
- logger.info("Read from ZK registry: AM host = %s, AM secured port =
%s" % (amHost, amSecuredPort))
self.hostname = amHost
self.secured_port = amSecuredPort
self.config.set(AgentConfig.SERVER_SECTION, "hostname",
self.hostname)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/448d598f/slider-agent/src/main/python/agent/Registry.py
--
diff --git a/slider-agent/src/main/python/agent/Registry.py
b/slide
[09/12] git commit: SLIDER-364 changed rpm directories to 755
SLIDER-364 changed rpm directories to 755
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38e6d343
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38e6d343
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38e6d343
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 38e6d343f1170450379559f05e6a9ddecf55d0d4
Parents: f155250
Author: Billie Rinaldi
Authored: Wed Aug 27 07:26:12 2014 -0700
Committer: Billie Rinaldi
Committed: Wed Aug 27 07:26:12 2014 -0700
--
slider-assembly/pom.xml | 8
1 file changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38e6d343/slider-assembly/pom.xml
--
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index 6c047e7..dbbd722 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -231,8 +231,8 @@
${rpm.basedir}
-
- 644
+
+ 755
${rpm.username}
${rpm.groupname}
@@ -258,8 +258,8 @@
${rpm.libdir}
-
- 644
+
+ 755
${rpm.username}
${rpm.groupname}
[11/12] git commit: SLIDER-149 ... in sync with YARN-913 change to return full list of endpoint addresses
SLIDER-149 ... in sync with YARN-913 change to return full list of endpoint
addresses
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e0854cf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e0854cf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e0854cf9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e0854cf9cf81da249394ad6d912ae86166d5d561
Parents: a052888
Author: Steve Loughran
Authored: Thu Aug 28 12:18:35 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 12:18:35 2014 +0100
--
.../slider/core/registry/retrieve/RegistryRetriever.java | 8
.../apache/slider/agent/standalone/TestYarnRegistryAM.groovy | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e0854cf9/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 001bf66..377b0b0 100644
---
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -79,13 +79,13 @@ public class RegistryRetriever {
Endpoint internal = record.getInternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-internalConfigurationURL = RegistryTypeUtils.retrieveAddressUriType(
-internal);
+internalConfigurationURL = RegistryTypeUtils.retrieveAddressesUriType(
+internal).get(0);
Endpoint external = record.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-externalConfigurationURL = RegistryTypeUtils.retrieveAddressUriType(
-external);
+externalConfigurationURL = RegistryTypeUtils.retrieveAddressesUriType(
+external).get(0);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e0854cf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
index 7bdf927..8ab5c6c 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
@@ -160,13 +160,13 @@ class TestYarnRegistryAM extends AgentMiniClusterTestBase
{
def registryEndpoint = serviceInstance.getExternalEndpoint(
CustomRegistryConstants.REGISTRY_REST_API)
assert registryEndpoint != null
-def registryURL = RegistryTypeUtils.retrieveAddressURL(registryEndpoint)
+def registryURL =
RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
describe("Registry WADL @ $registryURL")
def publisherEndpoint = serviceInstance.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_REST_API)
-def publisherURL = RegistryTypeUtils.retrieveAddressURL(publisherEndpoint)
+def publisherURL =
RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
def publisher = publisherURL.toString()
describe("Publisher")
[12/12] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c692461b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c692461b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c692461b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: c692461b715df7f2ef28df7e0b6af30773f6db3b Parents: e0854cf b126e7c Author: Steve Loughran Authored: Thu Aug 28 12:19:09 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 28 12:19:09 2014 +0100 -- app-packages/app-pkg-template/appConfig.json| 2 +- app-packages/app-pkg-template/metainfo.xml | 8 +- app-packages/hbase-win/README.txt | 34 ++ app-packages/hbase-win/appConfig.json | 44 +++ .../hbase-win/configuration/hbase-env.xml | 54 +++ .../hbase-win/configuration/hbase-log4j.xml | 143 +++ .../hbase-win/configuration/hbase-policy.xml| 53 +++ .../hbase-win/configuration/hbase-site.xml | 370 +++ app-packages/hbase-win/jmx_metrics.json | 56 +++ app-packages/hbase-win/metainfo.xml | 129 +++ ...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 + .../hbase-win/package/scripts/__init__.py | 19 + .../hbase-win/package/scripts/functions.py | 40 ++ app-packages/hbase-win/package/scripts/hbase.py | 66 .../hbase-win/package/scripts/hbase_master.py | 63 .../package/scripts/hbase_regionserver.py | 63 .../hbase-win/package/scripts/hbase_service.py | 48 +++ .../hbase-win/package/scripts/params.py | 81 .../hbase-win/package/scripts/status_params.py | 26 ++ .../package/templates/hbase_client_jaas.conf.j2 | 22 ++ .../package/templates/hbase_master_jaas.conf.j2 | 25 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 25 ++ app-packages/hbase-win/resources.json | 21 ++ app-packages/hbase/appConfig.json | 3 +- app-packages/hbase/package/scripts/hbase.py | 2 + app-packages/memcached-win/appConfig.json | 6 +- app-packages/memcached-win/metainfo.xml | 8 +- app-packages/memcached/appConfig.json | 2 +- app-packages/memcached/metainfo.xml | 8 +- app-packages/storm/appConfig.json | 2 +- slider-agent/src/main/python/agent/Constants.py | 1 + .../src/main/python/agent/Controller.py | 1 - slider-agent/src/main/python/agent/Registry.py | 9 +- slider-agent/src/main/python/agent/main.py | 60 +-- slider-agent/src/main/python/kazoo/client.py| 6 +- .../src/main/python/kazoo/handlers/threading.py | 6 +- .../src/main/python/kazoo/handlers/utils.py | 87 - .../main/python/kazoo/protocol/connection.py| 16 +- .../src/main/python/kazoo/tests/test_client.py | 6 +- .../main/python/kazoo/tests/test_connection.py | 55 +-- .../core/providers/__init__.py | 14 +- .../core/providers/windows/system.py| 74 +++- .../core/providers/windows/tarball.py | 46 +++ .../libraries/functions/check_process_status.py | 44 ++- .../libraries/script/script.py | 2 +- slider-agent/src/test/python/agent/TestMain.py | 19 +- slider-assembly/pom.xml | 8 +- .../providers/agent/AgentProviderService.java | 5 + 48 files changed, 1762 insertions(+), 137 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c692461b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java --
[3/3] git commit: Merge branch 'feature/SLIDER-36_Remove_custom_SliderAmIpFilter_for_REST_APIs_in_AM' into develop
Merge branch 'feature/SLIDER-36_Remove_custom_SliderAmIpFilter_for_REST_APIs_in_AM' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93e2b52f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93e2b52f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93e2b52f Branch: refs/heads/develop Commit: 93e2b52f92b0013172197fba5d9c140d976742bb Parents: b126e7c dcae064 Author: Steve Loughran Authored: Thu Aug 28 15:50:08 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 28 15:50:08 2014 +0100 -- .../org/apache/slider/common/SliderKeys.java| 7 + .../server/appmaster/SliderAppMaster.java | 8 +- .../web/SliderAmFilterInitializer.java | 71 --- .../server/appmaster/web/SliderAmIpFilter.java | 155 -- .../appmaster/web/TestSliderAmFilter.java | 200 --- 5 files changed, 9 insertions(+), 432 deletions(-) --
[2/3] git commit: SLIDER-36 delete custom IP filter and tests
SLIDER-36 delete custom IP filter and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dcae064c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dcae064c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dcae064c
Branch: refs/heads/develop
Commit: dcae064caa2dc636560821adcc222f038c17270e
Parents: 265c478
Author: Steve Loughran
Authored: Thu Aug 28 15:49:40 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 15:49:40 2014 +0100
--
.../web/SliderAmFilterInitializer.java | 72 ---
.../server/appmaster/web/SliderAmIpFilter.java | 161 ---
.../appmaster/web/TestSliderAmFilter.java | 201 ---
3 files changed, 434 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dcae064c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
deleted file mode 100644
index df6cea4..000
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.slider.server.appmaster.web;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.http.FilterInitializer;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Deprecated
-public class SliderAmFilterInitializer extends FilterInitializer {
- private static final String FILTER_NAME = "AM_PROXY_FILTER";
- private static final String FILTER_CLASS =
SliderAmIpFilter.class.getCanonicalName();
- private static final String HTTPS_PREFIX = "https://";;
- private static final String HTTP_PREFIX = "http://";;
- private Configuration configuration;
-
- public static final String NAME =
-"org.apache.slider.server.appmaster.web.SliderAmFilterInitializer";
-
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
-configuration = conf;
-Map params = new HashMap();
-String proxy = WebAppUtils.getProxyHostAndPort(conf);
-String[] parts = proxy.split(":");
-params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);
-// todo: eventually call WebAppUtils.getHttpSchemePrefix
-params.put(SliderAmIpFilter.PROXY_URI_BASE, getHttpSchemePrefix()
-+ proxy + getApplicationWebProxyBase());
-params.put(SliderAmIpFilter.WS_CONTEXT_ROOT,
- conf.get(SliderAmIpFilter.WS_CONTEXT_ROOT));
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
- }
-
- @VisibleForTesting
- protected String getApplicationWebProxyBase() {
-return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
- }
-
- private String getHttpSchemePrefix() {
-return HttpConfig.Policy.HTTPS_ONLY ==
- HttpConfig.Policy.fromString(configuration
- .get(
-
YarnConfiguration.YARN_HTTP_POLICY_KEY,
-
YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
- ? HTTPS_PREFIX : HTTP_PREFIX;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dcae064c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
[1/3] git commit: SLIDER-36 remove custom IP filter from AM; mark as deprecated
Repository: incubator-slider
Updated Branches:
refs/heads/develop b126e7cbe -> 93e2b52f9
SLIDER-36 remove custom IP filter from AM; mark as deprecated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/265c4785
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/265c4785
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/265c4785
Branch: refs/heads/develop
Commit: 265c4785c9ddf7f7fc0394069ad957784d6d6abc
Parents: b126e7c
Author: Steve Loughran
Authored: Thu Aug 28 15:48:26 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 15:48:26 2014 +0100
--
.../src/main/java/org/apache/slider/common/SliderKeys.java | 7 +++
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 8 ++--
.../server/appmaster/web/SliderAmFilterInitializer.java | 1 +
.../apache/slider/server/appmaster/web/SliderAmIpFilter.java | 6 ++
.../slider/server/appmaster/web/TestSliderAmFilter.java | 1 +
5 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 38f55c2..1d0a6f5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -172,4 +172,11 @@ public interface SliderKeys extends SliderXmlConfKeys {
* Python specific
*/
String PYTHONPATH = "PYTHONPATH";
+
+
+ /**
+ * Name of the AM filter to use: {@value}
+ */
+ String AM_FILTER_NAME =
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5a84c39..53ce4f8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -126,8 +126,6 @@ import
org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
-import org.apache.slider.server.appmaster.web.SliderAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.SliderAmIpFilter;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
@@ -536,10 +534,8 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
}
Configuration serviceConf = getConfig();
-// Try to get the proper filtering of static resources through the yarn
proxy working
-serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS,
-SliderAmFilterInitializer.NAME);
-serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT + "|" +
WS_AGENT_CONTEXT_ROOT);
+// IP filtering
+serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
MapOperations globalInternalOptions = getGlobalInternalOptions();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
index 5fffa4a..df6cea4 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import java.util.HashMap;
import java.util.Map;
+@Deprecated
public class SliderAmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS =
SliderAmIpFilter.class.getCanonicalName();
git commit: SLIDER-372 Chaos monkey service to not be created if interval ==0; RenewingAction also rejects 0 intervals
Repository: incubator-slider
Updated Branches:
refs/heads/develop 671830d70 -> 0c4fb57fe
SLIDER-372 Chaos monkey service to not be created if interval ==0;
RenewingAction also rejects 0 intervals
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0c4fb57f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0c4fb57f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0c4fb57f
Branch: refs/heads/develop
Commit: 0c4fb57fefb97cfc1ef99165758f25b931ad617b
Parents: 671830d
Author: Steve Loughran
Authored: Thu Aug 28 16:30:45 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 16:31:23 2014 +0100
--
.../apache/slider/server/appmaster/SliderAppMaster.java | 10 +-
.../slider/server/appmaster/actions/RenewingAction.java | 1 +
.../server/appmaster/monkey/ChaosMonkeyService.java | 4 ++--
.../agent/standalone/TestStandaloneAMMonkeyRestart.groovy | 4
4 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8e781f0..51c3b93 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1817,6 +1817,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
0);
+if (monkeyInterval == 0) {
+ log.debug(
+ "Chaos monkey not configured with a time interval...not enabling");
+ return false;
+}
log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
@@ -1838,8 +1843,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// and schedule it
if (monkey.schedule(monkeyInterval, TimeUnit.SECONDS)) {
log.info("Chaos Monkey is running");
+ return true;
+} else {
+ log.info("Chaos monkey not started");
+ return false;
}
-return true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
index 40c1021..1164df9 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
@@ -64,6 +64,7 @@ public class RenewingAction extends
AsyncAction {
// slightly superfluous as the super init above checks these
values...retained
// in case that code is ever changed
Preconditions.checkArgument(action != null, "null actions");
+Preconditions.checkArgument(interval > 0, "invalid interval: " + interval);
this.action = action;
this.interval = interval;
this.timeUnit = timeUnit;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index fa288af..f7b1eb7 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -94,10 +94,10 @@ public class ChaosMonkeyService extends AbstractService {
* Schedule the monkey
* @param time interval
* @param timeUnit time unit
- * @return true if it was scheduled (i.e. 1+ action)
+ * @return true if it was scheduled (i.e. 1+ action) and interval > 0
*/
public boolean schedule(long time, TimeUnit timeUnit) {
-if (!chaosEntries.isEmpty()) {
+if (time > 0 && !chaosEntries.isEmpty()) {
queues.schedule(getChaosAction(time, timeUnit));
git commit: SLIDER-371 test teardown warning of deletion failures ... fix root cause in SliderUtils
Repository: incubator-slider
Updated Branches:
refs/heads/develop e4d136e15 -> 4d5783e12
SLIDER-371 test teardown warning of deletion failures ... fix root cause in
SliderUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4d5783e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4d5783e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4d5783e1
Branch: refs/heads/develop
Commit: 4d5783e12e3f2a43bfe02e15bf789d34341af0f0
Parents: e4d136e
Author: Steve Loughran
Authored: Thu Aug 28 16:45:28 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 18:28:46 2014 +0100
--
.../apache/slider/common/tools/SliderUtils.java | 55 +++-
.../agent/AgentMiniClusterTestBase.groovy | 2 +-
2 files changed, 31 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d5783e1/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 188b7d9..a864878 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1460,35 +1460,40 @@ public final class SliderUtils {
String entry)
throws IOException {
InputStream is = null;
-FSDataInputStream appStream = fs.open(appPath);
-ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
-ZipArchiveEntry zipEntry;
-boolean done = false;
-while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
- if (entry.equals(zipEntry.getName())) {
-int size = (int) zipEntry.getSize();
-if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
-offset += zis.read(content, offset, size - offset);
- }
- is = new ByteArrayInputStream(content);
-} else {
- log.debug("Size unknown. Reading {}", zipEntry.getName());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- while (true) {
-int byteRead = zis.read();
-if (byteRead == -1) {
- break;
+FSDataInputStream appStream = null;
+try {
+ appStream = fs.open(appPath);
+ ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
+ ZipArchiveEntry zipEntry;
+ boolean done = false;
+ while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
+if (entry.equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+log.info("Reading {} of size {}", zipEntry.getName(),
zipEntry.getSize());
+byte[] content = new byte[size];
+int offset = 0;
+while (offset < size) {
+ offset += zis.read(content, offset, size - offset);
}
-baos.write(byteRead);
+is = new ByteArrayInputStream(content);
+ } else {
+log.debug("Size unknown. Reading {}", zipEntry.getName());
+ByteArrayOutputStream baos = new ByteArrayOutputStream();
+while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+break;
+ }
+ baos.write(byteRead);
+}
+is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(baos.toByteArray());
+ done = true;
}
-done = true;
}
+} finally {
+ IOUtils.closeStream(appStream);
}
return is;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d5783e1/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index a957481..4566d9d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -85,7 +85,7 @@ extends YarnZKMiniClusterTestBase {
try {
FileUtils.deleteDirectory(tempRoot);
} catch (IOException e) {
-log.war
[1/2] git commit: SLIDER-375 TestLongLivedProcess needed windows command translation
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows [created]
fbaea34f3
SLIDER-375 TestLongLivedProcess needed windows command translation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/637293a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/637293a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/637293a2
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 637293a2164def30f076fc7195046eb67941f4b6
Parents: 4d5783e
Author: Steve Loughran
Authored: Thu Aug 28 19:15:14 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 19:15:14 2014 +0100
--
.../workflow/ProcessCommandFactory.java | 21 ++--
.../services/workflow/TestLongLivedProcess.java | 6 ++
.../workflow/WorkflowServiceTestBase.java | 3 ++-
3 files changed, 19 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/637293a2/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List ls(File dir) {
-List commands = new ArrayList(5);
-commands.add("ls");
-commands.add("-1");
-commands.add(dir.getAbsolutePath());
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+} else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+}
return commands;
}
@@ -61,8 +66,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List env() {
-List commands = new ArrayList(1);
-commands.add("env");
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+} else {
+ commands = Arrays.asList("cmd", "/c", "set");
+}
return commands;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/637293a2/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..9019124 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -33,11 +33,9 @@ import java.util.List;
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log =
LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/637293a2/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Ca
[2/2] git commit: SLIDER-375 skip bits of TestPublisherURIs on windows
SLIDER-375 skip bits of TestPublisherURIs on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fbaea34f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fbaea34f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fbaea34f
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: fbaea34f3c7d2c9682c90c6dcd423e010dd51f83
Parents: 637293a
Author: Steve Loughran
Authored: Fri Aug 29 12:16:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 12:16:28 2014 +0100
--
.../publisher/TestPublisherRestResources.groovy | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbaea34f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 21881be..0f2014d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -134,13 +135,14 @@ class TestPublisherRestResources extends AgentTestBase {
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
-log.info("Classpath URIs: {}", uris)
-// check for some expected classpath elements
-assert uris.any {it =~ /curator-x-discovery/}
-assert uris.any {it =~ /hadoop-yarn-api/}
-assert uris.any {it =~ /hadoop-hdfs/}
-// and a negative test...
-assert !uris.any {it =~ /foo-bar/}
+if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+}
}
public String toFileURI(File filename) {
git commit: SLIDER-375 test getting its args wrong on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows fbaea34f3
-> 3ef13fb70
SLIDER-375 test getting its args wrong on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3ef13fb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3ef13fb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3ef13fb7
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 3ef13fb70f31dc71a268b8ec3e8213621c9cd78d
Parents: fbaea34
Author: Steve Loughran
Authored: Fri Aug 29 12:25:19 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 12:25:19 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3ef13fb7/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 1cd8de6..4e7b5b9 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
],
true, true,
true)
git commit: SLIDER-375 path fixup for windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows 3ef13fb70
-> 3c4f9f768
SLIDER-375 path fixup for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c4f9f76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c4f9f76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c4f9f76
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 3c4f9f7686aedf06f3a59a90e3b1c3087f5301f6
Parents: 3ef13fb
Author: Steve Loughran
Authored: Fri Aug 29 12:42:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 12:42:28 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 8 +---
.../apache/slider/providers/agent/TestAgentEcho.groovy| 6 +++---
.../registry/curator/TestRegistryRestResources.groovy | 6 +++---
.../web/rest/publisher/TestPublisherRestResources.groovy | 10 +++---
.../groovy/org/apache/slider/test/SliderTestUtils.groovy | 9 +
5 files changed, 23 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c4f9f76/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 4e7b5b9..6720bda 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
@@ -163,4 +163,6 @@ class TestAgentAMManagementWS extends AgentTestBase {
assert RegistrationStatus.FAILED == response.getResponseStatus();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c4f9f76/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..a7b5fe3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -72,9 +72,9 @@ class TestAgentEcho extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c4f9f76/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a1e5aa..0b77adc 100644
---
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -88,9 +88,9 @@ class TestRegistryRestResources extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slide
[1/6] git commit: SLIDER-366. App package clean up - remove extra information from appConfig.json for memcached.
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows 3c4f9f768
-> aceab6ba9 (forced update)
SLIDER-366. App package clean up - remove extra information from appConfig.json
for memcached.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f292a16e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f292a16e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f292a16e
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: f292a16ebfed4d75711f87577f4cde63698e3704
Parents: 4d5783e
Author: Sumit Mohanty
Authored: Thu Aug 28 12:51:49 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 28 12:51:49 2014 -0700
--
app-packages/memcached/appConfig.json | 2 --
1 file changed, 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f292a16e/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index 7a97f98..c335959 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -19,8 +19,6 @@
"components": {
"slider-appmaster": {
"jvm.heapsize": "256M"
-},
-"MEMCACHED": {
}
}
}
[2/6] git commit: SLIDER-258. Jenkins build continue to hang on python unit tests
SLIDER-258. Jenkins build continue to hang on python unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8bfd66e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8bfd66e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8bfd66e8
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 8bfd66e813635aa93a9551c121622b408d41b518
Parents: f292a16
Author: Sumit Mohanty
Authored: Thu Aug 28 15:37:31 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 28 15:37:31 2014 -0700
--
.../src/test/python/agent/TestRegistration.py | 14 +-
slider-agent/src/test/python/unitTests.py | 1 +
2 files changed, 2 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8bfd66e8/slider-agent/src/test/python/agent/TestRegistration.py
--
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py
b/slider-agent/src/test/python/agent/TestRegistration.py
index f91fe29..7b3c875 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -37,15 +37,6 @@ class TestRegistration(TestCase):
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
-try:
- os.mkdir(ver_dir)
-except OSError as exception:
- if exception.errno != errno.EEXIST:
-raise
-pass
-ver_file = os.path.join(ver_dir, "version")
-with open(ver_file, "w") as text_file:
- text_file.write("1.3.0")
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, 1)
@@ -54,7 +45,7 @@ class TestRegistration(TestCase):
self.assertEquals(data['publicHostname'] != "", True, "publicHostname
should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp
should not be empty")
-self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not
be empty")
+self.assertEquals(data['agentVersion'], '1', "agentVersion should not be
empty")
self.assertEquals(data['actualState'], State.INIT, "actualState should not
be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should
not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be
empty")
@@ -66,8 +57,5 @@ class TestRegistration(TestCase):
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("log_dir"))
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("app_task_dir"))
-os.remove(ver_file)
-os.removedirs(ver_dir)
-
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8bfd66e8/slider-agent/src/test/python/unitTests.py
--
diff --git a/slider-agent/src/test/python/unitTests.py
b/slider-agent/src/test/python/unitTests.py
index e3f2d7c..b01438e 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -56,6 +56,7 @@ def all_tests_suite():
files_list += os.listdir(src_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
+ files_list.sort()
tests_list = []
logger.info('TESTS
LIST:-')
[6/6] git commit: SLIDER-375 path fixup for windows
SLIDER-375 path fixup for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aceab6ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aceab6ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aceab6ba
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: aceab6ba942fb5e6c338570e030b4e09dce609eb
Parents: 26ec0d0
Author: Steve Loughran
Authored: Fri Aug 29 12:42:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 8 +---
.../apache/slider/providers/agent/TestAgentEcho.groovy| 6 +++---
.../registry/curator/TestRegistryRestResources.groovy | 6 +++---
.../web/rest/publisher/TestPublisherRestResources.groovy | 10 +++---
.../groovy/org/apache/slider/test/SliderTestUtils.groovy | 9 +
5 files changed, 23 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 4e7b5b9..6720bda 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
@@ -163,4 +163,6 @@ class TestAgentAMManagementWS extends AgentTestBase {
assert RegistrationStatus.FAILED == response.getResponseStatus();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..a7b5fe3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -72,9 +72,9 @@ class TestAgentEcho extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a1e5aa..0b77adc 100644
---
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -88,9 +88,9 @@ class TestRegistryRestResources extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" +
[5/6] git commit: SLIDER-375 test getting its args wrong on windows
SLIDER-375 test getting its args wrong on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26ec0d08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26ec0d08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26ec0d08
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 26ec0d084a16a7a6c5d27f946fa48451047263aa
Parents: c40e480
Author: Steve Loughran
Authored: Fri Aug 29 12:25:19 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/26ec0d08/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 1cd8de6..4e7b5b9 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
],
true, true,
true)
[4/6] git commit: SLIDER-375 TestLongLivedProcess needed windows command translation
SLIDER-375 TestLongLivedProcess needed windows command translation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6797222c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6797222c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6797222c
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: 6797222c92b887ffadb066251b5886787bb4cf44
Parents: 8bfd66e
Author: Steve Loughran
Authored: Thu Aug 28 19:15:14 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../workflow/ProcessCommandFactory.java | 21 ++--
.../services/workflow/TestLongLivedProcess.java | 6 ++
.../workflow/WorkflowServiceTestBase.java | 3 ++-
3 files changed, 19 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List ls(File dir) {
-List commands = new ArrayList(5);
-commands.add("ls");
-commands.add("-1");
-commands.add(dir.getAbsolutePath());
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+} else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+}
return commands;
}
@@ -61,8 +66,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List env() {
-List commands = new ArrayList(1);
-commands.add("env");
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+} else {
+ commands = Arrays.asList("cmd", "/c", "set");
+}
return commands;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..9019124 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -33,11 +33,9 @@ import java.util.List;
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log =
LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Callable;
/**
@@ -122,7 +123,7 @@ public abstract class WorkflowServiceTestBase extends
Assert {
boolean found = false;
StringB
[3/6] git commit: SLIDER-375 skip bits of TestPublisherURIs on windows
SLIDER-375 skip bits of TestPublisherURIs on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c40e480e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c40e480e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c40e480e
Branch: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows
Commit: c40e480e32d647451a87a80dcba01533b99c0d4f
Parents: 6797222
Author: Steve Loughran
Authored: Fri Aug 29 12:16:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../publisher/TestPublisherRestResources.groovy | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c40e480e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 21881be..0f2014d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -134,13 +135,14 @@ class TestPublisherRestResources extends AgentTestBase {
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
-log.info("Classpath URIs: {}", uris)
-// check for some expected classpath elements
-assert uris.any {it =~ /curator-x-discovery/}
-assert uris.any {it =~ /hadoop-yarn-api/}
-assert uris.any {it =~ /hadoop-hdfs/}
-// and a negative test...
-assert !uris.any {it =~ /foo-bar/}
+if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+}
}
public String toFileURI(File filename) {
[3/5] git commit: SLIDER-375 test getting its args wrong on windows
SLIDER-375 test getting its args wrong on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26ec0d08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26ec0d08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26ec0d08
Branch: refs/heads/develop
Commit: 26ec0d084a16a7a6c5d27f946fa48451047263aa
Parents: c40e480
Author: Steve Loughran
Authored: Fri Aug 29 12:25:19 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/26ec0d08/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 1cd8de6..4e7b5b9 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
],
true, true,
true)
[5/5] git commit: Merge branch 'feature/SLIDER-375_slider-core_tests_failing_on_windows' into develop
Merge branch 'feature/SLIDER-375_slider-core_tests_failing_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c91d5ecd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c91d5ecd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c91d5ecd Branch: refs/heads/develop Commit: c91d5ecd3a8b78d218bd02a5e2e7c9e7fefbc8cf Parents: 8bfd66e aceab6b Author: Steve Loughran Authored: Fri Aug 29 13:19:52 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 29 13:19:52 2014 +0100 -- .../agent/TestAgentAMManagementWS.groovy| 8 +++--- .../slider/providers/agent/TestAgentEcho.groovy | 6 ++--- .../curator/TestRegistryRestResources.groovy| 6 ++--- .../publisher/TestPublisherRestResources.groovy | 26 +--- .../apache/slider/test/SliderTestUtils.groovy | 9 +++ .../workflow/ProcessCommandFactory.java | 21 +++- .../services/workflow/TestLongLivedProcess.java | 6 ++--- .../workflow/WorkflowServiceTestBase.java | 3 ++- 8 files changed, 51 insertions(+), 34 deletions(-) --
[1/5] git commit: SLIDER-375 skip bits of TestPublisherURIs on windows
Repository: incubator-slider
Updated Branches:
refs/heads/develop 8bfd66e81 -> c91d5ecd3
SLIDER-375 skip bits of TestPublisherURIs on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c40e480e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c40e480e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c40e480e
Branch: refs/heads/develop
Commit: c40e480e32d647451a87a80dcba01533b99c0d4f
Parents: 6797222
Author: Steve Loughran
Authored: Fri Aug 29 12:16:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../publisher/TestPublisherRestResources.groovy | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c40e480e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 21881be..0f2014d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -134,13 +135,14 @@ class TestPublisherRestResources extends AgentTestBase {
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
-log.info("Classpath URIs: {}", uris)
-// check for some expected classpath elements
-assert uris.any {it =~ /curator-x-discovery/}
-assert uris.any {it =~ /hadoop-yarn-api/}
-assert uris.any {it =~ /hadoop-hdfs/}
-// and a negative test...
-assert !uris.any {it =~ /foo-bar/}
+if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+}
}
public String toFileURI(File filename) {
[4/5] git commit: SLIDER-375 path fixup for windows
SLIDER-375 path fixup for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aceab6ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aceab6ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aceab6ba
Branch: refs/heads/develop
Commit: aceab6ba942fb5e6c338570e030b4e09dce609eb
Parents: 26ec0d0
Author: Steve Loughran
Authored: Fri Aug 29 12:42:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 8 +---
.../apache/slider/providers/agent/TestAgentEcho.groovy| 6 +++---
.../registry/curator/TestRegistryRestResources.groovy | 6 +++---
.../web/rest/publisher/TestPublisherRestResources.groovy | 10 +++---
.../groovy/org/apache/slider/test/SliderTestUtils.groovy | 9 +
5 files changed, 23 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 4e7b5b9..6720bda 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
@@ -163,4 +163,6 @@ class TestAgentAMManagementWS extends AgentTestBase {
assert RegistrationStatus.FAILED == response.getResponseStatus();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..a7b5fe3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -72,9 +72,9 @@ class TestAgentEcho extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a1e5aa..0b77adc 100644
---
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -88,9 +88,9 @@ class TestRegistryRestResources extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OP
[2/5] git commit: SLIDER-375 TestLongLivedProcess needed windows command translation
SLIDER-375 TestLongLivedProcess needed windows command translation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6797222c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6797222c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6797222c
Branch: refs/heads/develop
Commit: 6797222c92b887ffadb066251b5886787bb4cf44
Parents: 8bfd66e
Author: Steve Loughran
Authored: Thu Aug 28 19:15:14 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../workflow/ProcessCommandFactory.java | 21 ++--
.../services/workflow/TestLongLivedProcess.java | 6 ++
.../workflow/WorkflowServiceTestBase.java | 3 ++-
3 files changed, 19 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List ls(File dir) {
-List commands = new ArrayList(5);
-commands.add("ls");
-commands.add("-1");
-commands.add(dir.getAbsolutePath());
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+} else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+}
return commands;
}
@@ -61,8 +66,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List env() {
-List commands = new ArrayList(1);
-commands.add("env");
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+} else {
+ commands = Arrays.asList("cmd", "/c", "set");
+}
return commands;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..9019124 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -33,11 +33,9 @@ import java.util.List;
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log =
LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Callable;
/**
@@ -122,7 +123,7 @@ public abstract class WorkflowServiceTestBase extends
Assert {
boolean found = false;
StringBuilder builder = new StringBuilder();
for (
[05/21] git commit: Merge branch 'feature/SLIDER-36_Remove_custom_SliderAmIpFilter_for_REST_APIs_in_AM' into develop
Merge branch 'feature/SLIDER-36_Remove_custom_SliderAmIpFilter_for_REST_APIs_in_AM' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93e2b52f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93e2b52f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93e2b52f Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 93e2b52f92b0013172197fba5d9c140d976742bb Parents: b126e7c dcae064 Author: Steve Loughran Authored: Thu Aug 28 15:50:08 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 28 15:50:08 2014 +0100 -- .../org/apache/slider/common/SliderKeys.java| 7 + .../server/appmaster/SliderAppMaster.java | 8 +- .../web/SliderAmFilterInitializer.java | 71 --- .../server/appmaster/web/SliderAmIpFilter.java | 155 -- .../appmaster/web/TestSliderAmFilter.java | 200 --- 5 files changed, 9 insertions(+), 432 deletions(-) --
[18/21] git commit: SLIDER-375 test getting its args wrong on windows
SLIDER-375 test getting its args wrong on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26ec0d08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26ec0d08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26ec0d08
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 26ec0d084a16a7a6c5d27f946fa48451047263aa
Parents: c40e480
Author: Steve Loughran
Authored: Fri Aug 29 12:25:19 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/26ec0d08/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 1cd8de6..4e7b5b9 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
+ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
],
true, true,
true)
[17/21] git commit: SLIDER-375 path fixup for windows
SLIDER-375 path fixup for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aceab6ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aceab6ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aceab6ba
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: aceab6ba942fb5e6c338570e030b4e09dce609eb
Parents: 26ec0d0
Author: Steve Loughran
Authored: Fri Aug 29 12:42:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:19:00 2014 +0100
--
.../slider/providers/agent/TestAgentAMManagementWS.groovy | 8 +---
.../apache/slider/providers/agent/TestAgentEcho.groovy| 6 +++---
.../registry/curator/TestRegistryRestResources.groovy | 6 +++---
.../web/rest/publisher/TestPublisherRestResources.groovy | 10 +++---
.../groovy/org/apache/slider/test/SliderTestUtils.groovy | 9 +
5 files changed, 23 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 4e7b5b9..6720bda 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@ class TestAgentAMManagementWS extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, app_def_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_CONF,
agt_conf_path.absoluteFile.toURI().toString(),
-ARG_OPTION, AGENT_VERSION,
agt_ver_path.absoluteFile.toURI().toString(),
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
@@ -163,4 +163,6 @@ class TestAgentAMManagementWS extends AgentTestBase {
assert RegistrationStatus.FAILED == response.getResponseStatus();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..a7b5fe3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -72,9 +72,9 @@ class TestAgentEcho extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
-ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aceab6ba/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a1e5aa..0b77adc 100644
---
a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -88,9 +88,9 @@ class TestRegistryRestResources extends AgentTestBase {
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
-ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
-ARG_OPTION, AGENT_CONF, "file://" + agt_
[19/21] git commit: Merge branch 'feature/SLIDER-375_slider-core_tests_failing_on_windows' into develop
Merge branch 'feature/SLIDER-375_slider-core_tests_failing_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c91d5ecd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c91d5ecd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c91d5ecd Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: c91d5ecd3a8b78d218bd02a5e2e7c9e7fefbc8cf Parents: 8bfd66e aceab6b Author: Steve Loughran Authored: Fri Aug 29 13:19:52 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 29 13:19:52 2014 +0100 -- .../agent/TestAgentAMManagementWS.groovy| 8 +++--- .../slider/providers/agent/TestAgentEcho.groovy | 6 ++--- .../curator/TestRegistryRestResources.groovy| 6 ++--- .../publisher/TestPublisherRestResources.groovy | 26 +--- .../apache/slider/test/SliderTestUtils.groovy | 9 +++ .../workflow/ProcessCommandFactory.java | 21 +++- .../services/workflow/TestLongLivedProcess.java | 6 ++--- .../workflow/WorkflowServiceTestBase.java | 3 ++- 8 files changed, 51 insertions(+), 34 deletions(-) --
[08/21] git commit: SLIDER-149 RegistryRetriever broken; add sleep to make test more robust to race conditions
SLIDER-149 RegistryRetriever broken; add sleep to make test more robust to race
conditions
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9708292b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9708292b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9708292b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 9708292b45189ab26c04d4fec31fd082c7a2e0b9
Parents: 5fad493
Author: Steve Loughran
Authored: Thu Aug 28 16:18:52 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 16:18:52 2014 +0100
--
.../registry/retrieve/RegistryRetriever.java| 20 +++-
.../agent/standalone/TestYarnRegistryAM.groovy | 5 -
2 files changed, 19 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9708292b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 377b0b0..a05d5be 100644
---
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.List;
/**
* Registry retriever.
@@ -78,14 +79,23 @@ public class RegistryRetriever {
public RegistryRetriever(ServiceRecord record) throws InvalidRecordException
{
Endpoint internal = record.getInternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-
-internalConfigurationURL = RegistryTypeUtils.retrieveAddressesUriType(
-internal).get(0);
+List addresses = RegistryTypeUtils.retrieveAddressesUriType(
+internal);
+if (addresses != null && !addresses.isEmpty()) {
+ internalConfigurationURL = addresses.get(0);
+} else {
+ internalConfigurationURL = "";
+}
Endpoint external = record.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-externalConfigurationURL = RegistryTypeUtils.retrieveAddressesUriType(
-external).get(0);
+addresses = RegistryTypeUtils.retrieveAddressesUriType(external);
+if (addresses != null && !addresses.isEmpty()) {
+ externalConfigurationURL = addresses.get(0);
+} else {
+ externalConfigurationURL = "";
+}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9708292b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
index 8ab5c6c..bdb5b27 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
@@ -125,8 +125,11 @@ class TestYarnRegistryAM extends AgentMiniClusterTestBase {
logReport(instance)
assert instance != null
-//switch to the ZK-based registry
+// sleep to allow registration to complete
+sleep(5000)
+
+
try {
def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
[16/21] git commit: SLIDER-375 skip bits of TestPublisherURIs on windows
SLIDER-375 skip bits of TestPublisherURIs on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c40e480e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c40e480e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c40e480e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c40e480e32d647451a87a80dcba01533b99c0d4f
Parents: 6797222
Author: Steve Loughran
Authored: Fri Aug 29 12:16:28 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../publisher/TestPublisherRestResources.groovy | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c40e480e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 21881be..0f2014d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.ClientResponse
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -134,13 +135,14 @@ class TestPublisherRestResources extends AgentTestBase {
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
-log.info("Classpath URIs: {}", uris)
-// check for some expected classpath elements
-assert uris.any {it =~ /curator-x-discovery/}
-assert uris.any {it =~ /hadoop-yarn-api/}
-assert uris.any {it =~ /hadoop-hdfs/}
-// and a negative test...
-assert !uris.any {it =~ /foo-bar/}
+if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+}
}
public String toFileURI(File filename) {
[10/21] git commit: SLIDER-369 keep accumulo ssl tests from erroring when jks files already exist
SLIDER-369 keep accumulo ssl tests from erroring when jks files already exist
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b336c576
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b336c576
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b336c576
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b336c576f53136ed6175dcb94402abcfb0942885
Parents: 0c4fb57
Author: Billie Rinaldi
Authored: Thu Aug 28 08:33:54 2014 -0700
Committer: Billie Rinaldi
Committed: Thu Aug 28 08:33:54 2014 -0700
--
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 28
1 file changed, 17 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b336c576/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
index 2818a3a..8a1b0d2 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
@@ -59,9 +59,11 @@ class AccumuloSSLTestBase extends AccumuloBasicIT {
System.out.println("provider after "+provider)
File rootKeyStoreFile = new File(TEST_APP_PKG_DIR, "root.jks")
-CertUtil.createRootKeyPair(rootKeyStoreFile.toString(),
- Property.INSTANCE_SECRET.toString(), trustStoreFile.toString(),
- Property.RPC_SSL_TRUSTSTORE_PASSWORD.toString(), provider);
+if (!rootKeyStoreFile.exists() && !trustStoreFile.exists()) {
+ CertUtil.createRootKeyPair(rootKeyStoreFile.toString(),
+Property.INSTANCE_SECRET.toString(), trustStoreFile.toString(),
+Property.RPC_SSL_TRUSTSTORE_PASSWORD.toString(), provider);
+}
AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG)
agentUploads.uploader.copyIfOutOfDate(trustStoreFile, new Path(certDir,
@@ -69,18 +71,22 @@ class AccumuloSSLTestBase extends AccumuloBasicIT {
for (node in getNodeList(conf)) {
File keyStoreFile = new File(TEST_APP_PKG_DIR, node + ".jks")
- CertUtil.createServerKeyPair(keyStoreFile.toString(),
-Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
-rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
-provider, node);
+ if (!keyStoreFile.exists()) {
+CertUtil.createServerKeyPair(keyStoreFile.toString(),
+ Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
+ rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
+ provider, node);
+ }
agentUploads.uploader.copyIfOutOfDate(keyStoreFile, new Path(certDir,
node + ".jks"), false)
}
-CertUtil.createServerKeyPair(clientKeyStoreFile.toString(),
- Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
- rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
- provider, InetAddress.getLocalHost().getHostName());
+if (!clientKeyStoreFile.exists()) {
+ CertUtil.createServerKeyPair(clientKeyStoreFile.toString(),
+Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
+rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
+provider, InetAddress.getLocalHost().getHostName());
+}
}
def getNodeList(Configuration conf) {
[03/21] git commit: SLIDER-36 remove custom IP filter from AM; mark as deprecated
SLIDER-36 remove custom IP filter from AM; mark as deprecated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/265c4785
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/265c4785
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/265c4785
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 265c4785c9ddf7f7fc0394069ad957784d6d6abc
Parents: b126e7c
Author: Steve Loughran
Authored: Thu Aug 28 15:48:26 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 15:48:26 2014 +0100
--
.../src/main/java/org/apache/slider/common/SliderKeys.java | 7 +++
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 8 ++--
.../server/appmaster/web/SliderAmFilterInitializer.java | 1 +
.../apache/slider/server/appmaster/web/SliderAmIpFilter.java | 6 ++
.../slider/server/appmaster/web/TestSliderAmFilter.java | 1 +
5 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 38f55c2..1d0a6f5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -172,4 +172,11 @@ public interface SliderKeys extends SliderXmlConfKeys {
* Python specific
*/
String PYTHONPATH = "PYTHONPATH";
+
+
+ /**
+ * Name of the AM filter to use: {@value}
+ */
+ String AM_FILTER_NAME =
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5a84c39..53ce4f8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -126,8 +126,6 @@ import
org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
-import org.apache.slider.server.appmaster.web.SliderAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.SliderAmIpFilter;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
@@ -536,10 +534,8 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
}
Configuration serviceConf = getConfig();
-// Try to get the proper filtering of static resources through the yarn
proxy working
-serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS,
-SliderAmFilterInitializer.NAME);
-serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT + "|" +
WS_AGENT_CONTEXT_ROOT);
+// IP filtering
+serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
MapOperations globalInternalOptions = getGlobalInternalOptions();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/265c4785/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
index 5fffa4a..df6cea4 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import java.util.HashMap;
import java.util.Map;
+@Deprecated
public class SliderAmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS =
SliderAmIpFilter.class.getCanonicalName();
http://git-wip-us.apache.org/repos/asf/incubator-s
[01/21] git commit: SLIDER-304 initial token renewal code for review
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry c692461b7 ->
c5fb4f040
SLIDER-304 initial token renewal code for review
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0bd6c6cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0bd6c6cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0bd6c6cf
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0bd6c6cf93b098358c252355e2fbde7e87bd2fe7
Parents: d3e4bf2
Author: Jon Maron
Authored: Mon Aug 25 18:30:20 2014 -0400
Committer: Jon Maron
Committed: Mon Aug 25 18:30:20 2014 -0400
--
.../server/appmaster/SliderAppMaster.java | 88 ---
.../appmaster/actions/RenewingAction.java | 35 ++-
.../security/FsDelegationTokenManager.java | 257 +++
.../TestFsDelegationTokenManager.groovy | 244 ++
4 files changed, 584 insertions(+), 40 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0bd6c6cf/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b4515f1..33ad1df 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -133,6 +133,7 @@ import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.security.CertificateManager;
+import org.apache.slider.server.services.security.FsDelegationTokenManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
@@ -351,6 +352,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
private final QueueService actionQueues = new QueueService();
private String agentOpsUrl;
private String agentStatusUrl;
+ private FsDelegationTokenManager fsDelegationTokenManager;
/**
* Service Constructor
@@ -417,8 +419,8 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
-executorService.execute(new QueueExecutor(this, actionQueues));
executorService.execute(actionQueues);
+executorService.execute(new QueueExecutor(this, actionQueues));
}
/* === */
@@ -583,36 +585,6 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
appAttemptID.toString());
-UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
-Credentials credentials =
- currentUser.getCredentials();
-DataOutputBuffer dob = new DataOutputBuffer();
-credentials.writeTokenStorageToStream(dob);
-dob.close();
-// Now remove the AM->RM token so that containers cannot access it.
-Iterator> iter = credentials.getAllTokens().iterator();
-while (iter.hasNext()) {
- Token token = iter.next();
- log.info("Token {}", token.getKind());
- if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
-iter.remove();
- }
-}
-allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-
-// set up secret manager
-secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
-
-// if not a secure cluster, extract the username -it will be
-// propagated to workers
-if (!UserGroupInformation.isSecurityEnabled()) {
- hadoop_user_name = System.getenv(HADOOP_USER_NAME);
- service_user_name = hadoop_user_name;
- log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name);
-} else {
- service_user_name = UserGroupInformation.getCurrentUser().getUserName();
-}
-
Map envVars;
List liveContainers;
/**
@@ -636,6 +608,9 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
nmClientAsync = new NMClientAsyncImpl("nmclient", this);
deployChildService(nmClientAsync);
+ // set up secret manager
+ secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
+
//bring up the Slider RPC service
[14/21] git commit: SLIDER-258. Jenkins build continue to hang on python unit tests
SLIDER-258. Jenkins build continue to hang on python unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8bfd66e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8bfd66e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8bfd66e8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8bfd66e813635aa93a9551c121622b408d41b518
Parents: f292a16
Author: Sumit Mohanty
Authored: Thu Aug 28 15:37:31 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 28 15:37:31 2014 -0700
--
.../src/test/python/agent/TestRegistration.py | 14 +-
slider-agent/src/test/python/unitTests.py | 1 +
2 files changed, 2 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8bfd66e8/slider-agent/src/test/python/agent/TestRegistration.py
--
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py
b/slider-agent/src/test/python/agent/TestRegistration.py
index f91fe29..7b3c875 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -37,15 +37,6 @@ class TestRegistration(TestCase):
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
-try:
- os.mkdir(ver_dir)
-except OSError as exception:
- if exception.errno != errno.EEXIST:
-raise
-pass
-ver_file = os.path.join(ver_dir, "version")
-with open(ver_file, "w") as text_file:
- text_file.write("1.3.0")
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, 1)
@@ -54,7 +45,7 @@ class TestRegistration(TestCase):
self.assertEquals(data['publicHostname'] != "", True, "publicHostname
should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp
should not be empty")
-self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not
be empty")
+self.assertEquals(data['agentVersion'], '1', "agentVersion should not be
empty")
self.assertEquals(data['actualState'], State.INIT, "actualState should not
be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should
not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be
empty")
@@ -66,8 +57,5 @@ class TestRegistration(TestCase):
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("log_dir"))
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("app_task_dir"))
-os.remove(ver_file)
-os.removedirs(ver_dir)
-
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8bfd66e8/slider-agent/src/test/python/unitTests.py
--
diff --git a/slider-agent/src/test/python/unitTests.py
b/slider-agent/src/test/python/unitTests.py
index e3f2d7c..b01438e 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -56,6 +56,7 @@ def all_tests_suite():
files_list += os.listdir(src_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
+ files_list.sort()
tests_list = []
logger.info('TESTS
LIST:-')
[11/21] git commit: SLIDER-368 change app package names to standard maven artifact names
SLIDER-368 change app package names to standard maven artifact names
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e4d136e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e4d136e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e4d136e1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e4d136e152161fa4c7a8f648773797d8b2541867
Parents: b336c57
Author: Billie Rinaldi
Authored: Thu Aug 28 08:36:17 2014 -0700
Committer: Billie Rinaldi
Committed: Thu Aug 28 08:36:17 2014 -0700
--
app-packages/accumulo/pom.xml | 2 +-
app-packages/hbase/pom.xml| 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e4d136e1/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 40d6ba7..548ced8 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -30,7 +30,7 @@
package-tmp
-
apache-slider-accumulo-${accumulo.version}-app-package-${project.version}
+
${project.artifactId}-${project.version}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e4d136e1/app-packages/hbase/pom.xml
--
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 7dede6c..442b0f2 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -29,7 +29,7 @@
Slider HBase App Package
package-tmp
-
apache-slider-hbase-${hbase.version}-app-package-${project.version}
+
${project.artifactId}-${project.version}
[09/21] git commit: SLIDER-372 Chaos monkey service to not be created if interval ==0; RenewingAction also rejects 0 intervals
SLIDER-372 Chaos monkey service to not be created if interval ==0;
RenewingAction also rejects 0 intervals
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0c4fb57f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0c4fb57f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0c4fb57f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0c4fb57fefb97cfc1ef99165758f25b931ad617b
Parents: 671830d
Author: Steve Loughran
Authored: Thu Aug 28 16:30:45 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 16:31:23 2014 +0100
--
.../apache/slider/server/appmaster/SliderAppMaster.java | 10 +-
.../slider/server/appmaster/actions/RenewingAction.java | 1 +
.../server/appmaster/monkey/ChaosMonkeyService.java | 4 ++--
.../agent/standalone/TestStandaloneAMMonkeyRestart.groovy | 4
4 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8e781f0..51c3b93 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1817,6 +1817,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
0);
+if (monkeyInterval == 0) {
+ log.debug(
+ "Chaos monkey not configured with a time interval...not enabling");
+ return false;
+}
log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
@@ -1838,8 +1843,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// and schedule it
if (monkey.schedule(monkeyInterval, TimeUnit.SECONDS)) {
log.info("Chaos Monkey is running");
+ return true;
+} else {
+ log.info("Chaos monkey not started");
+ return false;
}
-return true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
index 40c1021..1164df9 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
@@ -64,6 +64,7 @@ public class RenewingAction extends
AsyncAction {
// slightly superfluous as the super init above checks these
values...retained
// in case that code is ever changed
Preconditions.checkArgument(action != null, "null actions");
+Preconditions.checkArgument(interval > 0, "invalid interval: " + interval);
this.action = action;
this.interval = interval;
this.timeUnit = timeUnit;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c4fb57f/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index fa288af..f7b1eb7 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -94,10 +94,10 @@ public class ChaosMonkeyService extends AbstractService {
* Schedule the monkey
* @param time interval
* @param timeUnit time unit
- * @return true if it was scheduled (i.e. 1+ action)
+ * @return true if it was scheduled (i.e. 1+ action) and interval > 0
*/
public boolean schedule(long time, TimeUnit timeUnit) {
-if (!chaosEntries.isEmpty()) {
+if (time > 0 && !chaosEntries.isEmpty()) {
queues.schedule(getChaosAction(time, timeUnit));
return true;
} else {
http://git-wip-us.apa
[07/21] git commit: Merge branch 'feature/SLIDER-304_hdfs_token_renewal' into develop
Merge branch 'feature/SLIDER-304_hdfs_token_renewal' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/671830d7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/671830d7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/671830d7 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 671830d70bc0efa19e7d50b2b8228db4f42388aa Parents: 93e2b52 c939ac6 Author: Jon Maron Authored: Thu Aug 28 11:16:07 2014 -0400 Committer: Jon Maron Committed: Thu Aug 28 11:16:07 2014 -0400 -- .../server/appmaster/SliderAppMaster.java | 84 +++--- .../appmaster/actions/RenewingAction.java | 35 ++- .../security/FsDelegationTokenManager.java | 257 +++ .../TestFsDelegationTokenManager.groovy | 244 ++ 4 files changed, 581 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/671830d7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[06/21] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5fad4934 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5fad4934 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5fad4934 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 5fad4934a9bcf1412e5714b5d519f747254b3cf9 Parents: c692461 93e2b52 Author: Steve Loughran Authored: Thu Aug 28 15:50:44 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 28 15:50:44 2014 +0100 -- .../org/apache/slider/common/SliderKeys.java| 7 + .../server/appmaster/SliderAppMaster.java | 8 +- .../web/SliderAmFilterInitializer.java | 71 --- .../server/appmaster/web/SliderAmIpFilter.java | 155 -- .../appmaster/web/TestSliderAmFilter.java | 200 --- 5 files changed, 9 insertions(+), 432 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5fad4934/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[21/21] git commit: SLIDER-149 add PersistencePolicy
SLIDER-149 add PersistencePolicy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c5fb4f04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c5fb4f04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c5fb4f04
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c5fb4f040bd0727426405815a6aba8542c07aa2d
Parents: 4cbc9fe
Author: Steve Loughran
Authored: Fri Aug 29 17:49:03 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 17:49:03 2014 +0100
--
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c5fb4f04/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a2545ab..e80a362 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import
org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import
org.apache.hadoop.yarn.registry.server.services.ResourceManagerRegistryService;
@@ -165,9 +166,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import static
org.apache.slider.server.appmaster.web.rest.RestPaths.WS_AGENT_CONTEXT_ROOT;
-import static
org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
-
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -989,7 +987,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
/**
* Handler for {@link RegisterComponentInstance action}
- * Register/re-register a component (that is already in the app state
+ * Register/re-register an ephemeral container that is already in the app
state
* @param id the component
* @param description
*/
@@ -1004,7 +1002,8 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
ServiceRecord container = new ServiceRecord(
cid,
-description);
+description,
+PersistencePolicies.EPHEMERAL);
try {
yarnRegistryOperations.putComponent(cid, container, true);
} catch (IOException e) {
[12/21] git commit: SLIDER-371 test teardown warning of deletion failures ... fix root cause in SliderUtils
SLIDER-371 test teardown warning of deletion failures ... fix root cause in
SliderUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4d5783e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4d5783e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4d5783e1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4d5783e12e3f2a43bfe02e15bf789d34341af0f0
Parents: e4d136e
Author: Steve Loughran
Authored: Thu Aug 28 16:45:28 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 18:28:46 2014 +0100
--
.../apache/slider/common/tools/SliderUtils.java | 55 +++-
.../agent/AgentMiniClusterTestBase.groovy | 2 +-
2 files changed, 31 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d5783e1/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 188b7d9..a864878 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1460,35 +1460,40 @@ public final class SliderUtils {
String entry)
throws IOException {
InputStream is = null;
-FSDataInputStream appStream = fs.open(appPath);
-ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
-ZipArchiveEntry zipEntry;
-boolean done = false;
-while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
- if (entry.equals(zipEntry.getName())) {
-int size = (int) zipEntry.getSize();
-if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
-offset += zis.read(content, offset, size - offset);
- }
- is = new ByteArrayInputStream(content);
-} else {
- log.debug("Size unknown. Reading {}", zipEntry.getName());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- while (true) {
-int byteRead = zis.read();
-if (byteRead == -1) {
- break;
+FSDataInputStream appStream = null;
+try {
+ appStream = fs.open(appPath);
+ ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
+ ZipArchiveEntry zipEntry;
+ boolean done = false;
+ while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
+if (entry.equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+log.info("Reading {} of size {}", zipEntry.getName(),
zipEntry.getSize());
+byte[] content = new byte[size];
+int offset = 0;
+while (offset < size) {
+ offset += zis.read(content, offset, size - offset);
}
-baos.write(byteRead);
+is = new ByteArrayInputStream(content);
+ } else {
+log.debug("Size unknown. Reading {}", zipEntry.getName());
+ByteArrayOutputStream baos = new ByteArrayOutputStream();
+while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+break;
+ }
+ baos.write(byteRead);
+}
+is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(baos.toByteArray());
+ done = true;
}
-done = true;
}
+} finally {
+ IOUtils.closeStream(appStream);
}
return is;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d5783e1/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index a957481..4566d9d 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -85,7 +85,7 @@ extends YarnZKMiniClusterTestBase {
try {
FileUtils.deleteDirectory(tempRoot);
} catch (IOException e) {
-log.warn("Failed to delete $tempRoot :$e", e)
+lo
[04/21] git commit: SLIDER-36 delete custom IP filter and tests
SLIDER-36 delete custom IP filter and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dcae064c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dcae064c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dcae064c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: dcae064caa2dc636560821adcc222f038c17270e
Parents: 265c478
Author: Steve Loughran
Authored: Thu Aug 28 15:49:40 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 28 15:49:40 2014 +0100
--
.../web/SliderAmFilterInitializer.java | 72 ---
.../server/appmaster/web/SliderAmIpFilter.java | 161 ---
.../appmaster/web/TestSliderAmFilter.java | 201 ---
3 files changed, 434 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dcae064c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
deleted file mode 100644
index df6cea4..000
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.slider.server.appmaster.web;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.http.FilterInitializer;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Deprecated
-public class SliderAmFilterInitializer extends FilterInitializer {
- private static final String FILTER_NAME = "AM_PROXY_FILTER";
- private static final String FILTER_CLASS =
SliderAmIpFilter.class.getCanonicalName();
- private static final String HTTPS_PREFIX = "https://";;
- private static final String HTTP_PREFIX = "http://";;
- private Configuration configuration;
-
- public static final String NAME =
-"org.apache.slider.server.appmaster.web.SliderAmFilterInitializer";
-
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
-configuration = conf;
-Map params = new HashMap();
-String proxy = WebAppUtils.getProxyHostAndPort(conf);
-String[] parts = proxy.split(":");
-params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);
-// todo: eventually call WebAppUtils.getHttpSchemePrefix
-params.put(SliderAmIpFilter.PROXY_URI_BASE, getHttpSchemePrefix()
-+ proxy + getApplicationWebProxyBase());
-params.put(SliderAmIpFilter.WS_CONTEXT_ROOT,
- conf.get(SliderAmIpFilter.WS_CONTEXT_ROOT));
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
- }
-
- @VisibleForTesting
- protected String getApplicationWebProxyBase() {
-return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
- }
-
- private String getHttpSchemePrefix() {
-return HttpConfig.Policy.HTTPS_ONLY ==
- HttpConfig.Policy.fromString(configuration
- .get(
-
YarnConfiguration.YARN_HTTP_POLICY_KEY,
-
YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
- ? HTTPS_PREFIX : HTTP_PREFIX;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dcae064c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/
[15/21] git commit: SLIDER-375 TestLongLivedProcess needed windows command translation
SLIDER-375 TestLongLivedProcess needed windows command translation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6797222c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6797222c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6797222c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6797222c92b887ffadb066251b5886787bb4cf44
Parents: 8bfd66e
Author: Steve Loughran
Authored: Thu Aug 28 19:15:14 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 13:18:59 2014 +0100
--
.../workflow/ProcessCommandFactory.java | 21 ++--
.../services/workflow/TestLongLivedProcess.java | 6 ++
.../workflow/WorkflowServiceTestBase.java | 3 ++-
3 files changed, 19 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List ls(File dir) {
-List commands = new ArrayList(5);
-commands.add("ls");
-commands.add("-1");
-commands.add(dir.getAbsolutePath());
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+} else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+}
return commands;
}
@@ -61,8 +66,12 @@ public class ProcessCommandFactory {
* @return commands
*/
public List env() {
-List commands = new ArrayList(1);
-commands.add("env");
+List commands;
+if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+} else {
+ commands = Arrays.asList("cmd", "/c", "set");
+}
return commands;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..9019124 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -33,11 +33,9 @@ import java.util.List;
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log =
LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6797222c/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
---
a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++
b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Callable;
/**
@@ -122,7 +123,7 @@ public abstract class WorkflowServiceTestBase extends
Assert {
boolean found = false;
StringBuilde
[02/21] git commit: Merge branch 'develop' into feature/SLIDER-304_hdfs_token_renewal
Merge branch 'develop' into feature/SLIDER-304_hdfs_token_renewal Conflicts: slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c939ac60 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c939ac60 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c939ac60 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: c939ac609fd9db1fa3c076ec64e9cf777124cbab Parents: 0bd6c6c 38e6d34 Author: Jon Maron Authored: Wed Aug 27 14:16:05 2014 -0400 Committer: Jon Maron Committed: Wed Aug 27 14:16:05 2014 -0400 -- app-packages/accumulo/README.txt| 28 +- app-packages/accumulo/pom.xml | 5 + .../slider/accumulo/CustomAuthenticator.java| 4 +- app-packages/app-pkg-template/metainfo.xml | 8 +- app-packages/hbase-win/README.txt | 34 ++ app-packages/hbase-win/appConfig.json | 44 +++ .../hbase-win/configuration/hbase-env.xml | 54 +++ .../hbase-win/configuration/hbase-log4j.xml | 143 +++ .../hbase-win/configuration/hbase-policy.xml| 53 +++ .../hbase-win/configuration/hbase-site.xml | 370 +++ app-packages/hbase-win/jmx_metrics.json | 56 +++ app-packages/hbase-win/metainfo.xml | 129 +++ ...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 + .../hbase-win/package/scripts/__init__.py | 19 + .../hbase-win/package/scripts/functions.py | 40 ++ app-packages/hbase-win/package/scripts/hbase.py | 66 .../hbase-win/package/scripts/hbase_master.py | 63 .../package/scripts/hbase_regionserver.py | 63 .../hbase-win/package/scripts/hbase_service.py | 48 +++ .../hbase-win/package/scripts/params.py | 81 .../hbase-win/package/scripts/status_params.py | 26 ++ .../package/templates/hbase_client_jaas.conf.j2 | 22 ++ .../package/templates/hbase_master_jaas.conf.j2 | 25 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 25 ++ app-packages/hbase-win/resources.json | 21 ++ app-packages/hbase/appConfig.json | 3 +- app-packages/hbase/package/scripts/hbase.py | 2 + app-packages/memcached-win/appConfig.json | 6 +- app-packages/memcached-win/metainfo.xml | 8 +- app-packages/memcached/metainfo.xml | 8 +- slider-agent/src/main/python/agent/Constants.py | 1 + .../src/main/python/agent/Controller.py | 1 - slider-agent/src/main/python/agent/Registry.py | 9 +- slider-agent/src/main/python/agent/main.py | 60 +-- slider-agent/src/main/python/kazoo/client.py| 6 +- .../src/main/python/kazoo/handlers/threading.py | 6 +- .../src/main/python/kazoo/handlers/utils.py | 87 - .../main/python/kazoo/protocol/connection.py| 16 +- .../src/main/python/kazoo/tests/test_client.py | 6 +- .../main/python/kazoo/tests/test_connection.py | 55 +-- .../core/providers/__init__.py | 14 +- .../core/providers/windows/system.py| 74 +++- .../core/providers/windows/tarball.py | 46 +++ .../libraries/functions/check_process_status.py | 44 ++- .../libraries/script/script.py | 2 +- slider-agent/src/test/python/agent/TestMain.py | 19 +- slider-assembly/pom.xml | 8 +- .../providers/agent/AgentProviderService.java | 5 + .../server/appmaster/SliderAppMaster.java | 25 +- .../appmaster/monkey/ChaosMonkeyService.java| 46 ++- .../model/monkey/TestMockMonkey.groovy | 41 +- .../failures/TestKilledHBaseAM.groovy | 1 - 52 files changed, 1880 insertions(+), 163 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c939ac60/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[13/21] git commit: SLIDER-366. App package clean up - remove extra information from appConfig.json for memcached.
SLIDER-366. App package clean up - remove extra information from appConfig.json
for memcached.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f292a16e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f292a16e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f292a16e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f292a16ebfed4d75711f87577f4cde63698e3704
Parents: 4d5783e
Author: Sumit Mohanty
Authored: Thu Aug 28 12:51:49 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 28 12:51:49 2014 -0700
--
app-packages/memcached/appConfig.json | 2 --
1 file changed, 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f292a16e/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index 7a97f98..c335959 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -19,8 +19,6 @@
"components": {
"slider-appmaster": {
"jvm.heapsize": "256M"
-},
-"MEMCACHED": {
}
}
}
[20/21] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4cbc9fe7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4cbc9fe7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4cbc9fe7 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4cbc9fe7fe8f047c92a5a40062ef1f5ebe36 Parents: 9708292 c91d5ec Author: Steve Loughran Authored: Fri Aug 29 15:54:56 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 29 15:54:56 2014 +0100 -- app-packages/accumulo/pom.xml | 2 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 28 +- app-packages/hbase/pom.xml | 2 +- app-packages/memcached/appConfig.json | 2 - .../src/test/python/agent/TestRegistration.py | 14 +- slider-agent/src/test/python/unitTests.py | 1 + .../apache/slider/common/tools/SliderUtils.java | 55 ++-- .../server/appmaster/SliderAppMaster.java | 94 --- .../appmaster/actions/RenewingAction.java | 36 ++- .../appmaster/monkey/ChaosMonkeyService.java| 4 +- .../security/FsDelegationTokenManager.java | 257 +++ .../agent/AgentMiniClusterTestBase.groovy | 2 +- .../TestStandaloneAMMonkeyRestart.groovy| 4 - .../agent/TestAgentAMManagementWS.groovy| 8 +- .../slider/providers/agent/TestAgentEcho.groovy | 6 +- .../curator/TestRegistryRestResources.groovy| 6 +- .../publisher/TestPublisherRestResources.groovy | 26 +- .../TestFsDelegationTokenManager.groovy | 244 ++ .../apache/slider/test/SliderTestUtils.groovy | 9 + .../workflow/ProcessCommandFactory.java | 21 +- .../services/workflow/TestLongLivedProcess.java | 6 +- .../workflow/WorkflowServiceTestBase.java | 3 +- 22 files changed, 696 insertions(+), 134 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4cbc9fe7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java -- diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 6bd67d2,51c3b93..a2545ab --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@@ -363,7 -350,7 +364,8 @@@ public class SliderAppMaster extends Ab private final QueueService actionQueues = new QueueService(); private String agentOpsUrl; private String agentStatusUrl; + private YarnRegistryViewForProviders yarnRegistryOperations; + private FsDelegationTokenManager fsDelegationTokenManager; /** * Service Constructor http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4cbc9fe7/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy --
git commit: SLIDER-377: changing outcome if winutils.exe is not on the path
Repository: incubator-slider
Updated Branches:
refs/heads/develop c91d5ecd3 -> 686fdff35
SLIDER-377: changing outcome if winutils.exe is not on the path
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/686fdff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/686fdff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/686fdff3
Branch: refs/heads/develop
Commit: 686fdff358a978fdc56240e82c045dfc5ae0a591
Parents: c91d5ec
Author: Steve Loughran
Authored: Fri Aug 29 17:54:21 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 17:54:21 2014 +0100
--
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/686fdff3/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 5a091bf..5aef585 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -132,7 +132,12 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
@BeforeClass
public static void checkWindowsSupport() {
if (Shell.WINDOWS) {
- assertNotNull("winutils.exe not found", Shell.WINUTILS)
+// assertNotNull("winutils.exe not found", Shell.WINUTILS)
+ if (!Shell.WINUTILS) {
+log.error("winutils.exe not found")
+ }
+ def lib = System.getProperty("java.library.path")
+ log.debug("java.library.path = ${lib}")
}
}
git commit: using "is None" over "== None" in .py
Repository: incubator-slider
Updated Branches:
refs/heads/develop 686fdff35 -> 329cab03e
using "is None" over "== None" in .py
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/329cab03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/329cab03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/329cab03
Branch: refs/heads/develop
Commit: 329cab03e07759fdaea4d77737abe0a42d4f9947
Parents: 686fdff
Author: Steve Loughran
Authored: Fri Aug 29 19:09:07 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 19:09:07 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/329cab03/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index db4b881..be79973 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -147,9 +147,9 @@ def java(classname, args, classpath, jvm_opts_list):
"""
# split the JVM opts by space
# java = "/usr/bin/java"
- prg="java"
- if which("java")==None:
-prg=os.environ["JAVA_HOME"]+"/bin/java"
+ prg = "java"
+ if which("java") is None:
+prg = os.environ["JAVA_HOME"] + "/bin/java"
commandline = [prg]
commandline.extend(jvm_opts_list)
commandline.append("-classpath")
[05/10] git commit: SLIDER-379. Modify default appConfig.json for he HBase package
SLIDER-379. Modify default appConfig.json for he HBase package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ffbc3a3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ffbc3a3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ffbc3a3c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ffbc3a3c31d762d8635ab11394a608c74b080582
Parents: 11f6f2b
Author: Sumit Mohanty
Authored: Sat Aug 30 07:42:06 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Aug 30 07:42:06 2014 -0700
--
app-packages/hbase/appConfig.json | 76 +-
1 file changed, 38 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ffbc3a3c/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index d324e6d..7df0dc3 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -1,45 +1,45 @@
{
- "schema": "http://example.org/specification/v2.0.0";,
- "metadata": {
- },
- "global": {
-"application.def": "${app.package.name}.zip",
-"create.default.zookeeper.node": "true",
-"java_home": "/usr/jdk64/jdk1.7.0_45",
-"system_configs": "core-site",
+"schema": "http://example.org/specification/v2.0.0";,
+"metadata": {
+},
+"global": {
+"application.def": "${app.package.name}.zip",
+"create.default.zookeeper.node": "true",
+"java_home": "/usr/jdk64/jdk1.7.0_45",
+"system_configs": "core-site",
-"site.global.app_user": "yarn",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+"site.global.app_user": "yarn",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
-"site.global.hbase_instance_name": "instancename",
-"site.global.hbase_root_password": "secret",
-"site.global.user_group": "hadoop",
-"site.global.security_enabled": "false",
-"site.global.monitor_protocol": "http",
-"site.global.ganglia_server_host": "${NN_HOST}",
-"site.global.ganglia_server_port": "8667",
-"site.global.ganglia_server_id": "Application1",
-"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
-"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
-"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+"site.global.hbase_instance_name": "instancename",
+"site.global.hbase_root_password": "secret",
+"site.global.user_group": "hadoop",
+"site.global.security_enabled": "false",
+"site.global.monitor_protocol": "http",
+"site.global.ganglia_server_host": "${NN_HOST}",
+"site.global.ganglia_server_port": "8667",
+"site.global.ganglia_server_id": "Application1",
+"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
-"site.hbase-env.hbase_master_heapsize": "1024m",
-"site.hbase-env.hbase_regionserver_heapsize": "1024m",
+"site.hbase-env.hbase_master_heapsize": "1024m",
+"site.hbase-env.hbase_regionserver_heapsize": "1024m",
-"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
-"site.hbase-site.hbase.superuser": "yarn",
-"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
-"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
-"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
-"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
-"site.hbase-site.hbase.regionserver.info.port": "0",
-"site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.hbase-site.hbase.master.port": "0"
- },
- "components": {
-"slider-appmaster": {
- "jvm.heapsize": "256M"
+"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+"site.hbase-site.hbase.superuser": "yarn",
+"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+"site.hbase-site.hbase.regionserver.info.port": "0",
+"site.hbase-site.hbase.master.info.port":
"${HBASE_MASTER.ALLOCATED_PORT}",
+"site.hbase-site.hbase.regionserver.port": "0",
+"site.hbase-site.hbase.master.port": "0"
+},
+"component
[01/10] git commit: SLIDER-377: changing outcome if winutils.exe is not on the path
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry c5fb4f040 ->
6e883bf91
SLIDER-377: changing outcome if winutils.exe is not on the path
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/686fdff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/686fdff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/686fdff3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 686fdff358a978fdc56240e82c045dfc5ae0a591
Parents: c91d5ec
Author: Steve Loughran
Authored: Fri Aug 29 17:54:21 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 17:54:21 2014 +0100
--
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/686fdff3/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 5a091bf..5aef585 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -132,7 +132,12 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
@BeforeClass
public static void checkWindowsSupport() {
if (Shell.WINDOWS) {
- assertNotNull("winutils.exe not found", Shell.WINUTILS)
+// assertNotNull("winutils.exe not found", Shell.WINUTILS)
+ if (!Shell.WINUTILS) {
+log.error("winutils.exe not found")
+ }
+ def lib = System.getProperty("java.library.path")
+ log.debug("java.library.path = ${lib}")
}
}
[09/10] git commit: Revert "SLIDER-187. Use start/stop instead of freeze/thaw"
Revert "SLIDER-187. Use start/stop instead of freeze/thaw" This reverts commit 0350273fd96579c1179a1f74a7d144e525f71572. Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/33ccc65a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/33ccc65a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/33ccc65a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 33ccc65a512f261c81c2412ebe9cd07a280e7326 Parents: be548b8 Author: Sumit Mohanty Authored: Fri Aug 29 11:58:53 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 30 10:12:05 2014 -0700 -- README.md | 4 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy| 2 +- .../apache/slider/api/ClusterDescription.java | 2 +- .../org/apache/slider/client/SliderClient.java | 20 - .../common/SliderXMLConfKeysForTesting.java | 8 ++-- .../slider/common/params/ActionFreezeArgs.java | 10 ++--- .../slider/common/params/ActionThawArgs.java| 6 +-- .../apache/slider/common/params/ClientArgs.java | 4 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 14 +++--- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 14 +++--- .../agent/actions/TestActionExists.groovy | 2 +- .../agent/freezethaw/TestFreezeCommands.groovy | 20 - .../TestFreezeThawMasterlessAM.groovy | 8 ++-- .../freezethaw/TestFreezeUnknownCluster.groovy | 4 +- .../standalone/TestBuildStandaloneAM.groovy | 8 ++-- .../standalone/TestStandaloneAMDestroy.groovy | 8 ++-- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../slider/client/TestClientBadArgs.groovy | 2 +- .../slider/client/TestCommonArgParsing.groovy | 12 ++--- .../model/history/TestRoleHistoryRW.groovy | 2 +- .../slider/test/YarnMiniClusterTestBase.groovy | 20 ++--- .../src/test/resources/example-slider-test.xml | 8 ++-- .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 46 ++-- .../commands/UnknownClusterOperationsIT.groovy | 9 ++-- .../lifecycle/AgentClusterLifecycleIT.groovy| 42 +- .../src/test/manual/python/SliderTester.py | 4 +- .../FunctionalAccumuloM1T1GC1Mon1IT.groovy | 6 +-- .../accumulo/live/TestAccFreezeThaw.groovy | 2 +- .../funtest/HBaseClusterLifecycleIT.groovy | 38 .../TestFreezeThawClusterFromArchive.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 10 ++--- ...reezeReconfigureThawLiveRegionService.groovy | 12 ++--- .../TestFreezeThawLiveRegionService.groovy | 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 4 +- src/test/clusters/sandbox/operations.md | 20 - src/test/clusters/ubuntu-secure/operations.md | 14 +++--- 40 files changed, 204 insertions(+), 197 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/33ccc65a/README.md -- diff --git a/README.md b/README.md index a25b83a..b91e477 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ Slider is a YARN application to deploy existing distributed applications on YARN monitor them and make them larger or smaller as desired -even while the cluster is running. -Clusters can be stopped and restarted later; the distribution +Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution of the deployed application across the YARN cluster is persisted -enabling -a best-effort placement close to the previous locations on a cluster start. +a best-effort placement close to the previous locations on a cluster thaw. Applications which remember the previous placement of data (such as HBase) can exhibit fast start-up times from this feature. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/33ccc65a/slider-assembly/src/main/bash/README.md -- diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md index d818aeb..a4b7b08 100644 --- a/slider-assembly/src/main/bash/README.md +++ b/slider-assembly/src/main/bash/README.md @@ -79,7 +79,7 @@ Tear Down - * slider_destroy will do the following - 1. Stop the slider application based on provided name + 1. Freeze the slider application based on provided name 2. Destory the slider application based on provided name * The following args are required http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/33c
[03/10] git commit: SLIDER-380. Add ability for a application to specify what system configs they need SLIDER-379. HBase package should by default set hbase.master.port to 0
SLIDER-380. Add ability for a application to specify what system configs they
need
SLIDER-379. HBase package should by default set hbase.master.port to 0
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4c076c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4c076c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4c076c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ae4c076cd24f86f6ad1ef665e1375fa42d3b12f0
Parents: 329cab0
Author: Sumit Mohanty
Authored: Fri Aug 29 20:54:56 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Aug 29 20:55:05 2014 -0700
--
app-packages/hbase/appConfig.json | 4 +++-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 20 ++--
.../agent/TestAgentProviderService.java | 17 +
4 files changed, 39 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 9c37ed9..d324e6d 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -6,6 +6,7 @@
"application.def": "${app.package.name}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
+"system_configs": "core-site",
"site.global.app_user": "yarn",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
@@ -33,7 +34,8 @@
"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
"site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
-"site.hbase-site.hbase.regionserver.port": "0"
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 419fa1a..1622367 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -85,6 +85,7 @@ public interface AgentKeys {
String JAVA_HOME = "java_home";
String PACKAGE_LIST = "package_list";
+ String SYSTEM_CONFIGS = "system_configs";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4c076c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 669a19a..be5b1ac 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1348,9 +1348,10 @@ public class AgentProviderService extends
AbstractProviderService implements
new TreeMap>();
Map tokens = getStandardTokenMap(appConf);
-List configs = getApplicationConfigurationTypes();
+Set configs = new HashSet();
+configs.addAll(getApplicationConfigurationTypes());
+configs.addAll(getSystemConfigurationsRequested(appConf));
-//Add global
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
configurations, tokens, containerId);
@@ -1374,6 +1375,21 @@ public class AgentProviderService extends
AbstractProviderService implements
}
@VisibleForTesting
+ protected List getSystemConfigurationsRequested(ConfTreeOperations
appConf) {
+List configList = new ArrayList();
+
+String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS);
+if (configTypes != null && configTypes.length() > 0) {
+ String[] configs = configTypes.split(",");
+ for(String config :configs)
+ configList.add(config.trim());
+}
+
+return new ArrayList(new HashSet(configList));
+ }
+
+
+ @Visibl
[10/10] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e883bf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e883bf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e883bf9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6e883bf91de781fa8ec5ca7bb1f98f2b9fd8a0dd
Parents: c5fb4f0 82cf1f0
Author: Steve Loughran
Authored: Mon Sep 1 16:55:59 2014 +0100
Committer: Steve Loughran
Committed: Mon Sep 1 16:55:59 2014 +0100
--
README.md | 4 +-
app-packages/hbase/appConfig.json | 74 +-
app-packages/hbase/pom.xml | 2 +-
.../src/main/python/agent/Controller.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 10 +-
slider-agent/src/main/python/agent/Register.py | 5 +-
.../src/test/python/agent/TestRegistration.py | 5 +-
slider-assembly/src/main/bash/README.md | 2 +-
slider-assembly/src/main/bash/slider_destroy| 4 +-
slider-assembly/src/main/scripts/slider.py | 6 +-
.../org/apache/slider/client/SliderClient.java | 10 +-
.../slider/common/params/ActionFreezeArgs.java | 2 +-
.../common/params/LaunchArgsAccessor.java | 2 +-
.../slider/common/params/SliderActions.java | 8 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 147 ---
.../server/appmaster/state/NodeEntry.java | 2 +-
.../server/appmaster/state/RoleHistory.java | 6 +-
.../appmaster/web/rest/agent/Register.java | 13 ++
.../agent/actions/TestActionExists.groovy | 2 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 10 +-
.../TestFreezeThawMasterlessAM.groovy | 8 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 2 +-
.../standalone/TestStandaloneAMDestroy.groovy | 6 +-
.../agent/standalone/TestYarnRegistryAM.groovy | 2 +-
.../slider/client/TestCommonArgParsing.groovy | 2 +-
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 13 +-
.../agent/TestAgentProviderService.java | 35 -
.../src/test/resources/example-slider-test.xml | 4 +-
.../funtest/framework/CommandTestBase.groovy| 2 +-
.../lifecycle/AgentClusterLifecycleIT.groovy| 8 +-
.../src/test/manual/python/SliderTester.py | 4 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../funtest/HBaseClusterLifecycleIT.groovy | 8 +-
.../TestFreezeThawClusterFromArchive.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
.../TestFreezeThawLiveRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 4 +-
src/test/clusters/sandbox/operations.md | 20 +--
src/test/clusters/ubuntu-secure/operations.md | 14 +-
43 files changed, 286 insertions(+), 184 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --cc
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a91a817,61866fb..4bfc718
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -582,11 -589,11 +592,11 @@@ public class AgentProviderService exten
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn,
roleName);
-
+
// and update registration entries
if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
-TimeUnit.MILLISECONDS));
+ queueAccess.put(new RegisterComponentInstance(instance.getId(),
+ roleName, 0, TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
--
[06/10] git commit: SLIDER-187. Use start/stop instead of freeze/thaw
SLIDER-187. Use start/stop instead of freeze/thaw Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4c106134 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4c106134 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4c106134 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4c1061341a7a9c47b84103e69b4e2054a1599276 Parents: 33ccc65 Author: Sumit Mohanty Authored: Fri Aug 29 17:39:20 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 30 10:12:05 2014 -0700 -- README.md | 4 ++-- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy| 4 ++-- .../org/apache/slider/client/SliderClient.java | 10 +- .../slider/common/params/ActionFreezeArgs.java | 2 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 8 .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +++--- .../agent/actions/TestActionExists.groovy | 2 +- .../agent/freezethaw/TestFreezeCommands.groovy | 10 +- .../TestFreezeThawMasterlessAM.groovy | 8 .../freezethaw/TestFreezeUnknownCluster.groovy | 2 +- .../standalone/TestBuildStandaloneAM.groovy | 2 +- .../standalone/TestStandaloneAMDestroy.groovy | 6 +++--- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../slider/client/TestCommonArgParsing.groovy | 2 +- .../model/history/TestRoleHistoryRW.groovy | 4 ++-- .../slider/test/YarnMiniClusterTestBase.groovy | 6 +++--- .../src/test/resources/example-slider-test.xml | 4 ++-- .../funtest/framework/CommandTestBase.groovy| 2 +- .../lifecycle/AgentClusterLifecycleIT.groovy| 8 .../src/test/manual/python/SliderTester.py | 4 ++-- .../accumulo/live/TestAccFreezeThaw.groovy | 4 ++-- .../funtest/HBaseClusterLifecycleIT.groovy | 8 .../TestFreezeThawClusterFromArchive.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 2 +- ...reezeReconfigureThawLiveRegionService.groovy | 4 ++-- .../TestFreezeThawLiveRegionService.groovy | 4 ++-- .../minicluster/live/TestTwoLiveClusters.groovy | 4 ++-- src/test/clusters/sandbox/operations.md | 20 ++-- src/test/clusters/ubuntu-secure/operations.md | 14 +++--- 32 files changed, 82 insertions(+), 82 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c106134/README.md -- diff --git a/README.md b/README.md index b91e477..a25b83a 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ Slider is a YARN application to deploy existing distributed applications on YARN monitor them and make them larger or smaller as desired -even while the cluster is running. -Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution +Clusters can be stopped and restarted later; the distribution of the deployed application across the YARN cluster is persisted -enabling -a best-effort placement close to the previous locations on a cluster thaw. +a best-effort placement close to the previous locations on a cluster start. Applications which remember the previous placement of data (such as HBase) can exhibit fast start-up times from this feature. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c106134/slider-assembly/src/main/bash/README.md -- diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md index a4b7b08..d818aeb 100644 --- a/slider-assembly/src/main/bash/README.md +++ b/slider-assembly/src/main/bash/README.md @@ -79,7 +79,7 @@ Tear Down - * slider_destroy will do the following - 1. Freeze the slider application based on provided name + 1. Stop the slider application based on provided name 2. Destory the slider application based on provided name * The following args are required http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c106134/slider-assembly/src/main/bash/slider_destroy -- diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy index 9039751..7ca30b7 100755 --- a/slider-assembly/src/main/bash/slider_destroy +++ b/slider-assembly/src/main/bash/slider_destroy @@ -53,8 +53,8 @@ source $config # # Main # -echo -e "\n## Freezing app $app_name" -sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1 +echo
[04/10] git commit: SLIDER-378. Slider AM restart- Logfolders and componentinstancedata not getting published
SLIDER-378. Slider AM restart- Logfolders and componentinstancedata not getting
published
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/11f6f2b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/11f6f2b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/11f6f2b3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 11f6f2b3738f8b8c05be0324eec375e270ce1b3c
Parents: ae4c076
Author: Sumit Mohanty
Authored: Fri Aug 29 23:42:21 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Aug 29 23:42:21 2014 -0700
--
.../src/main/python/agent/Controller.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 10 +-
slider-agent/src/main/python/agent/Register.py | 5 +-
.../src/test/python/agent/TestRegistration.py | 5 +-
.../providers/agent/AgentProviderService.java | 134 +++
.../appmaster/web/rest/agent/Register.java | 13 ++
.../agent/TestAgentProviderService.java | 18 ++-
7 files changed, 123 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11f6f2b3/slider-agent/src/main/python/agent/Controller.py
--
diff --git a/slider-agent/src/main/python/agent/Controller.py
b/slider-agent/src/main/python/agent/Controller.py
index a3fb90d..11db21c 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -122,6 +122,7 @@ class Controller(threading.Thread):
self.componentActualState,
self.componentExpectedState,
self.actionQueue.customServiceOrchestrator.allocated_ports,
+ self.actionQueue.customServiceOrchestrator.log_folders,
id))
logger.info("Registering with the server at " + self.registerUrl +
" with data " + pprint.pformat(data))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11f6f2b3/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 15f1664..dd8e9b9 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -26,6 +26,7 @@ import sys
import socket
import posixpath
import platform
+import copy
from AgentConfig import AgentConfig
from AgentException import AgentException
from PythonExecutor import PythonExecutor
@@ -58,6 +59,7 @@ class CustomServiceOrchestrator():
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
self.allocated_ports = {}
+self.log_folders = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -133,15 +135,17 @@ class CustomServiceOrchestrator():
}
if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
- ret[Constants.ALLOCATED_PORTS] = allocated_ports
- self.allocated_ports = allocated_ports
+ ret[Constants.ALLOCATED_PORTS] = copy.deepcopy(allocated_ports)
+ ## Generally all ports are allocated at once but just in case
+ self.allocated_ports.update(allocated_ports)
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
- ret[Constants.FOLDERS] = {
+ self.log_folders = {
Constants.AGENT_LOG_ROOT: self.config.getLogPath(),
Constants.AGENT_WORK_ROOT: self.config.getWorkRootPath()
}
+ ret[Constants.FOLDERS] = copy.deepcopy(self.log_folders)
return ret
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11f6f2b3/slider-agent/src/main/python/agent/Register.py
--
diff --git a/slider-agent/src/main/python/agent/Register.py
b/slider-agent/src/main/python/agent/Register.py
index b59154f..c8246c7 100644
--- a/slider-agent/src/main/python/agent/Register.py
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -29,7 +29,7 @@ class Register:
def __init__(self, config):
self.config = config
- def build(self, actualState, expectedState, allocated_ports, id='-1'):
+ def build(self, actualState, expectedState, allocated_ports, log_folders,
id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
@@ -41,7 +41,8 @@ class Register:
'agentVersion': version,
'actualState': actualState,
'expectedState': expectedState,
-'allocatedPorts': allocated_ports
+
[07/10] git commit: SLIDER-379. Modify pom version for the HBase package
SLIDER-379. Modify pom version for the HBase package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/82cf1f01 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/82cf1f01 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/82cf1f01 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 82cf1f010fbea58557003216d01341f832d3d6ac Parents: 4c10613 Author: Sumit Mohanty Authored: Sat Aug 30 10:11:56 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 30 10:12:05 2014 -0700 -- app-packages/hbase/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/82cf1f01/app-packages/hbase/pom.xml -- diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml index 442b0f2..6f1da9c 100644 --- a/app-packages/hbase/pom.xml +++ b/app-packages/hbase/pom.xml @@ -19,7 +19,7 @@ org.apache.slider slider -0.41.0-incubating-SNAPSHOT +0.51.0-incubating-SNAPSHOT ../../pom.xml 4.0.0
[08/10] git commit: SLIDER-187. Use start/stop instead of freeze/thaw
SLIDER-187. Use start/stop instead of freeze/thaw Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be548b87 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be548b87 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be548b87 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: be548b8742324f2cde7ef48271ec3f19c53e8e5c Parents: ffbc3a3 Author: Sumit Mohanty Authored: Thu Aug 28 14:27:32 2014 -0700 Committer: Sumit Mohanty Committed: Sat Aug 30 10:12:05 2014 -0700 -- README.md | 4 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy| 2 +- .../apache/slider/api/ClusterDescription.java | 2 +- .../org/apache/slider/client/SliderClient.java | 20 - .../common/SliderXMLConfKeysForTesting.java | 8 ++-- .../slider/common/params/ActionFreezeArgs.java | 10 ++--- .../slider/common/params/ActionThawArgs.java| 6 +-- .../apache/slider/common/params/ClientArgs.java | 4 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 14 +++--- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 14 +++--- .../agent/actions/TestActionExists.groovy | 2 +- .../agent/freezethaw/TestFreezeCommands.groovy | 20 - .../TestFreezeThawMasterlessAM.groovy | 8 ++-- .../freezethaw/TestFreezeUnknownCluster.groovy | 4 +- .../standalone/TestBuildStandaloneAM.groovy | 8 ++-- .../standalone/TestStandaloneAMDestroy.groovy | 8 ++-- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../slider/client/TestClientBadArgs.groovy | 2 +- .../slider/client/TestCommonArgParsing.groovy | 12 ++--- .../model/history/TestRoleHistoryRW.groovy | 2 +- .../slider/test/YarnMiniClusterTestBase.groovy | 20 +++-- .../src/test/resources/example-slider-test.xml | 8 ++-- .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 46 ++-- .../commands/UnknownClusterOperationsIT.groovy | 9 ++-- .../lifecycle/AgentClusterLifecycleIT.groovy| 42 +- .../src/test/manual/python/SliderTester.py | 4 +- .../FunctionalAccumuloM1T1GC1Mon1IT.groovy | 6 +-- .../accumulo/live/TestAccFreezeThaw.groovy | 2 +- .../funtest/HBaseClusterLifecycleIT.groovy | 38 .../TestFreezeThawClusterFromArchive.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 10 ++--- ...reezeReconfigureThawLiveRegionService.groovy | 12 ++--- .../TestFreezeThawLiveRegionService.groovy | 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 4 +- src/test/clusters/sandbox/operations.md | 20 - src/test/clusters/ubuntu-secure/operations.md | 14 +++--- 40 files changed, 197 insertions(+), 204 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be548b87/README.md -- diff --git a/README.md b/README.md index b91e477..a25b83a 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ Slider is a YARN application to deploy existing distributed applications on YARN monitor them and make them larger or smaller as desired -even while the cluster is running. -Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution +Clusters can be stopped and restarted later; the distribution of the deployed application across the YARN cluster is persisted -enabling -a best-effort placement close to the previous locations on a cluster thaw. +a best-effort placement close to the previous locations on a cluster start. Applications which remember the previous placement of data (such as HBase) can exhibit fast start-up times from this feature. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be548b87/slider-assembly/src/main/bash/README.md -- diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md index a4b7b08..d818aeb 100644 --- a/slider-assembly/src/main/bash/README.md +++ b/slider-assembly/src/main/bash/README.md @@ -79,7 +79,7 @@ Tear Down - * slider_destroy will do the following - 1. Freeze the slider application based on provided name + 1. Stop the slider application based on provided name 2. Destory the slider application based on provided name * The following args are required http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be548b87/slider-assembly/src/main/bash/slider_destroy -
[02/10] git commit: using "is None" over "== None" in .py
using "is None" over "== None" in .py
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/329cab03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/329cab03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/329cab03
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 329cab03e07759fdaea4d77737abe0a42d4f9947
Parents: 686fdff
Author: Steve Loughran
Authored: Fri Aug 29 19:09:07 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 29 19:09:07 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/329cab03/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index db4b881..be79973 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -147,9 +147,9 @@ def java(classname, args, classpath, jvm_opts_list):
"""
# split the JVM opts by space
# java = "/usr/bin/java"
- prg="java"
- if which("java")==None:
-prg=os.environ["JAVA_HOME"]+"/bin/java"
+ prg = "java"
+ if which("java") is None:
+prg = os.environ["JAVA_HOME"] + "/bin/java"
commandline = [prg]
commandline.extend(jvm_opts_list)
commandline.append("-classpath")
[1/2] git commit: SLIDER-327 AM to move to optimized, queued Node review and refresh action
Repository: incubator-slider
Updated Branches:
refs/heads/develop b525287dd -> e7dd217a3
SLIDER-327 AM to move to optimized, queued Node review and refresh action
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88b665db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88b665db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88b665db
Branch: refs/heads/develop
Commit: 88b665db938325bf2c221614f7ffadefe71be632
Parents: 82cf1f0
Author: Steve Loughran
Authored: Tue Sep 2 21:01:50 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 2 21:01:50 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 9 +--
.../server/appmaster/RoleLaunchService.java | 8 +--
.../server/appmaster/SliderAppMaster.java | 66 ++--
.../server/appmaster/actions/ActionHalt.java| 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +-
.../appmaster/actions/ActionStartContainer.java | 8 ++-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 --
.../actions/ProviderReportedContainerLoss.java | 13 ++--
.../actions/ProviderStartupCompleted.java | 4 --
.../server/appmaster/actions/QueueAccess.java | 5 ++
.../server/appmaster/actions/QueueService.java | 11 +++-
.../actions/RegisterComponentInstance.java | 5 ++
.../actions/ReviewAndFlexApplicationSize.java | 43 +
.../actions/UnregisterComponentInstance.java| 7 ++-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy| 8 +--
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../flexing/TestClusterFlex1To1.groovy | 2 +-
20 files changed, 176 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f0da136..f6d761e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2002,12 +2002,9 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
log.info("Flexing running cluster");
SliderClusterProtocol appMaster = connect(instance);
SliderClusterOperations clusterOps = new
SliderClusterOperations(appMaster);
- if (clusterOps.flex(instanceDefinition.getResources())) {
-log.info("Cluster size updated");
-exitCode = EXIT_SUCCESS;
- } else {
-log.info("Requested size is the same as current size: no change");
- }
+ clusterOps.flex(instanceDefinition.getResources());
+ log.info("application instance size updated");
+ exitCode = EXIT_SUCCESS;
} else {
log.info("No running instance to update");
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index e8b6802..37824c8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
@@ -39,9 +38,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
/**
* A service for launching containers
@@ -215,9 +214,8 @@ public class RoleLaunchService
instance.roleId = role.id;
instance.environment = envDescription;
actionQueue.put(new ActionStartContainer("sta
[2/2] git commit: Merge branch 'feature/SLIDER-327_AM_to_move_to_optimized, _queued_Node_review_and_refresh_action' into develop
Merge branch 'feature/SLIDER-327_AM_to_move_to_optimized,_queued_Node_review_and_refresh_action' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7dd217a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7dd217a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7dd217a Branch: refs/heads/develop Commit: e7dd217a324c205826bea4ac040b808c4ad72d61 Parents: b525287 88b665d Author: Steve Loughran Authored: Tue Sep 2 21:02:29 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 2 21:02:29 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 9 +-- .../server/appmaster/RoleLaunchService.java | 8 +-- .../server/appmaster/SliderAppMaster.java | 66 ++-- .../server/appmaster/actions/ActionHalt.java| 6 +- .../appmaster/actions/ActionKillContainer.java | 20 +- .../appmaster/actions/ActionStartContainer.java | 8 ++- .../appmaster/actions/ActionStopSlider.java | 6 +- .../server/appmaster/actions/AsyncAction.java | 47 -- .../actions/ProviderReportedContainerLoss.java | 13 ++-- .../actions/ProviderStartupCompleted.java | 4 -- .../server/appmaster/actions/QueueAccess.java | 5 ++ .../server/appmaster/actions/QueueService.java | 11 +++- .../actions/RegisterComponentInstance.java | 5 ++ .../actions/ReviewAndFlexApplicationSize.java | 43 + .../actions/UnregisterComponentInstance.java| 7 ++- .../server/appmaster/monkey/ChaosEntry.java | 2 +- .../TestStandaloneAMMonkeyRestart.groovy| 8 +-- .../providers/accumulo/AccumuloTestBase.groovy | 4 +- .../minicluster/HBaseMiniClusterTestBase.groovy | 1 - .../flexing/TestClusterFlex1To1.groovy | 2 +- 20 files changed, 176 insertions(+), 99 deletions(-) --
svn commit: r1622109 - /incubator/slider/site/trunk/content/docs/manpage.md
Author: stevel Date: Tue Sep 2 20:04:57 2014 New Revision: 1622109 URL: http://svn.apache.org/r1622109 Log: SLIDER-327 AM to move to optimized, queued Node review and refresh action Modified: incubator/slider/site/trunk/content/docs/manpage.md Modified: incubator/slider/site/trunk/content/docs/manpage.md URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/manpage.md?rev=1622109&r1=1622108&r2=1622109&view=diff == --- incubator/slider/site/trunk/content/docs/manpage.md (original) +++ incubator/slider/site/trunk/content/docs/manpage.md Tue Sep 2 20:04:57 2014 @@ -302,11 +302,15 @@ Example: ### `flex [--component component count]* ` -Flex the number of workers in an application instance to the new value. If greater than before, new copies of the component will be requested. If less, component instances will be destroyed. +Flex the number of workers in an application instance to the new value. +If greater than before, new copies of the component will be requested. +If less, component instances will be destroyed. + +This operation has a return value of 0 if the change was submitted. Changes +are not immediate and depend on the availability of resources in the YARN +cluster -This operation has a return value of 0 if the size of a running instance was changed. - -It returns -1 if there is no running application instance, or the size of the flexed instance matches that of the original -in which case its state does not change. +It returns -1 if there is no running application instance Example
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-350-windows [created] e9a353c1a
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-375_slider-core_tests_failing_on_windows [deleted] aceab6ba9
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-353_killJavaProcesses_doesn't_work_on_windows [deleted] 38c2a138b
