SLIDER-171: connectivity tests to verify state of funtest client config
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fb73dbca Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fb73dbca Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fb73dbca Branch: refs/heads/develop Commit: fb73dbca80cd87c2880d7e46325a67c507f58ee0 Parents: a6a4680 Author: Steve Loughran <ste...@apache.org> Authored: Tue Jul 1 20:08:54 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Jul 1 20:08:54 2014 +0100 ---------------------------------------------------------------------- .../slider/client/SliderYarnClientImpl.java | 3 +- .../funtest/framework/CommandTestBase.groovy | 4 +- .../basic/TestClusterConnectivity.groovy | 96 ++++++++++++++++++++ .../clusters/c6401/slider/slider-client.xml | 14 ++- .../clusters/morzine/slider/slider-client.xml | 29 ++++-- 5 files changed, 132 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 0c83e0c..e7b492b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -19,6 +19,7 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -74,7 +75,7 @@ public class SliderYarnClientImpl extends YarnClientImpl { List<ApplicationReport> allApps = getApplications(types); List<ApplicationReport> results = new ArrayList<>(); for (ApplicationReport report : allApps) { - if (user == null || user.equals(report.getUser())) { + if (StringUtils.isEmpty(user) || user.equals(report.getUser())) { results.add(report); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 61ae804..08d352a 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -103,15 +103,13 @@ abstract class CommandTestBase extends SliderTestUtils { log.debug("Security enabled") SliderUtils.forceLogin() } else { - log.info "Security off, making cluster dirs broadly accessible" + log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY SliderShell.script = SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") - // now patch the settings with the path of the conf direcotry - } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy new file mode 100644 index 0000000..cf9cc9d --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.basic + +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.Path +import org.apache.hadoop.net.NetUtils +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.client.SliderYarnClientImpl +import org.apache.slider.common.SliderXmlConfKeys +import org.apache.slider.core.zk.ZookeeperUtils +import org.apache.slider.funtest.framework.CommandTestBase +import org.junit.Test + +@Slf4j +class TestClusterConnectivity extends CommandTestBase { + + @Test + public void testFileSystemUp() throws Throwable { + + def fs = clusterFS + def status = fs.listStatus(new Path("/")) + status.each {it -> log.info(it.path.toString())} + + } + + @Test + public void testZKBinding() throws Throwable { + def quorum = SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM) + assert quorum + def tuples = ZookeeperUtils.splitToHostsAndPortsStrictly(quorum); + tuples.each {it -> telnet(it.getHostText(), it.getPort())} + + } + + @Test + public void testRMTelnet() throws Throwable { + def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0) + telnet(rmAddr.hostName, rmAddr.port) + } + + @Test + public void testRMBinding() throws Throwable { + testRMTelnet() + SliderYarnClientImpl yarnClient = new SliderYarnClientImpl() + try { + SLIDER_CONFIG.setInt("ipc.client.connect.retry.interval",100) + SLIDER_CONFIG.setInt( + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,5000) + SLIDER_CONFIG.setInt( + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,50) + + yarnClient.init(SLIDER_CONFIG) + yarnClient.start(); + def instances = yarnClient.listInstances("") + instances.each {it -> log.info("Instance $it.applicationId")} + } finally { + yarnClient.stop() + } + } + + def telnet(String host, int port) { + assert host != "" + assert port != 0 + try { + def socket = new Socket(host, port); + } catch (IOException e) { + throw NetUtils.wrapException(host, port, "localhost", 0, e) + } +/* + + socket.withStreams { input, output -> + output << "echo testing ...\n" + def buffer = input.newReader().readLine() + println "response = ${buffer}" +*/ + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/src/test/clusters/c6401/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/c6401/slider/slider-client.xml b/src/test/clusters/c6401/slider/slider-client.xml index fe2caad..526881b 100644 --- a/src/test/clusters/c6401/slider/slider-client.xml +++ b/src/test/clusters/c6401/slider/slider-client.xml @@ -21,25 +21,31 @@ They are not passed to the AM --> <configuration> + + <property> + <name>hostname</name> + <value>c6401</value> + </property> + <property> <name>slider.client.resource.origin</name> - <value>configs/c6401/slider</value> + <value>configs/${hostname}/slider</value> <description>This is just for diagnostics</description> </property> <property> <name>yarn.resourcemanager.address</name> - <value>c6401:8050</value> + <value>${hostname}:8050</value> </property> <property> <name>fs.defaultFS</name> - <value>hdfs://c6401.ambari.apache.org:8020</value> + <value>hdfs://${hostname}.ambari.apache.org:8020</value> </property> <property> <name>slider.zookeeper.quorum</name> - <value>c6401:2181</value> + <value>${hostname}:2181</value> </property> <property> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/src/test/clusters/morzine/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/morzine/slider/slider-client.xml b/src/test/clusters/morzine/slider/slider-client.xml index d15d443..4d7ab41 100644 --- a/src/test/clusters/morzine/slider/slider-client.xml +++ b/src/test/clusters/morzine/slider/slider-client.xml @@ -17,28 +17,45 @@ limitations under the License. --> <!-- - Morzine is runing Hadoop on Windows + Properties set here are picked up in the client. + They are not passed to the AM --> +<configuration> + + <property> + <name>hostname</name> + <value>morzine</value> + </property> + + <property> + <name>slider.client.resource.origin</name> + <value>configs/${hostname}/slider</value> + <description>This is just for diagnostics</description> + </property> <property> <name>yarn.resourcemanager.address</name> - <value>morzine:8050</value> + <value>${hostname}:8032</value> </property> - + <property> <name>fs.defaultFS</name> - <value>hdfs://morzine:8020</value> + <value>hdfs://${hostname}:8020</value> </property> <property> <name>slider.zookeeper.quorum</name> - <value>morzine:2181</value> + <value>${hostname}:2181</value> </property> + <!-- + This is a windows path as picked up from + http://morzine:8088/conf + --> <property> <name>yarn.application.classpath</name> <value> - /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/* + %HADOOP_CONF_DIR%,%HADOOP_COMMON_HOME%/share/hadoop/common/*,%HADOOP_COMMON_HOME%/share/hadoop/common/lib/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/lib/* </value> </property>