Author: stevel Date: Fri May 16 14:51:27 2014 New Revision: 1595221 URL: http://svn.apache.org/r1595221 Log: SLIDER-62 add functional test for registry entries against hbase provider
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy incubator/slider/trunk/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy incubator/slider/trunk/src/site/markdown/developing/functional_tests.md incubator/slider/trunk/src/site/markdown/manpage.md incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java Fri May 16 14:51:27 2014 @@ -1971,6 +1971,8 @@ public class SliderClient extends Abstra registryArgs); } } catch (FileNotFoundException e) { + log.info("{}", e.toString()); + log.debug("{}", e); return EXIT_NOT_FOUND; } return EXIT_SUCCESS; @@ -2079,7 +2081,14 @@ public class SliderClient extends Abstra external); return published; } - + + /** + * write out the config + * @param published + * @param registryArgs + * @throws BadCommandArgumentsException + * @throws IOException + */ private void outputConfig(PublishedConfiguration published, ActionRegistryArgs registryArgs) throws BadCommandArgumentsException, @@ -2136,16 +2145,14 @@ public class SliderClient extends Abstra */ private ServiceInstanceData lookupInstance(String id, String serviceType) throws - UnknownApplicationInstanceException, - SliderException, IOException { try { CuratorServiceInstance<ServiceInstanceData> csi = getRegistry().queryForInstance(serviceType, id); if (csi == null) { - throw new UnknownApplicationInstanceException( - "instance %s of type %s not found", - id, serviceType); + throw new FileNotFoundException( + String.format("instance %s of type %s not found", + id, serviceType)); } return csi.getPayload(); } catch (IOException e) { Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java Fri May 16 14:51:27 2014 @@ -64,4 +64,11 @@ public interface SliderXMLConfKeysForTes String KEY_TEST_TEARDOWN_KILLALL = "slider.test.teardown.killall"; boolean DEFAULT_TEARDOWN_KILLALL = true; + + + /** + * Key for amount of RAM to request + */ + String KEY_TEST_YARN_RAM_REQUEST = "slider.test.yarn.ram"; + String DEFAULT_YARN_RAM_REQUEST = "192"; } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java Fri May 16 14:51:27 2014 @@ -23,6 +23,7 @@ import com.beust.jcommander.Parameters; import org.apache.slider.common.SliderKeys; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.slider.core.registry.docstore.ConfigFormat; import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY; import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY; @@ -41,6 +42,14 @@ import java.io.File; commandDescription = DESCRIBE_ACTION_REGISTRY) public class ActionRegistryArgs extends AbstractActionArgs { + + public ActionRegistryArgs() { + } + + public ActionRegistryArgs(String name) { + this.name = name; + } + @Override public String getActionName() { return ACTION_REGISTRY; @@ -81,7 +90,7 @@ public class ActionRegistryArgs extends //--format @Parameter(names = ARG_FORMAT, description = "Format for a response: [xml|json|properties]") - public String format; + public String format = ConfigFormat.XML.toString() ; @Parameter(names = {ARG_DEST}, @@ -124,11 +133,6 @@ public class ActionRegistryArgs extends throw new BadCommandArgumentsException("Argument " + ARG_DEST + " is only supported on 'get' operations"); } - if (is(format) && !is(getConf)) { - throw new BadCommandArgumentsException("Argument " + ARG_FORMAT - + " is only supported by " + ARG_GETCONF); - - } if (!list && !is(name)) { throw new BadCommandArgumentsException("Argument " + ARG_NAME +" missing"); Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy Fri May 16 14:51:27 2014 @@ -306,26 +306,31 @@ class TestStandaloneRegistryAM extends A registryArgs.list = false; registryArgs.listConf = false registryArgs.internal = false - registryArgs.format = "properties" def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG registryArgs.getConf = yarn_site_config - - + + //properties format + registryArgs.format = "properties" describe registryArgs.toString() + client.actionRegistry(registryArgs) + File outputDir = new File("target/test_standalone_registry_am/output") outputDir.mkdirs() + // create a new registry args with the defaults back in + registryArgs = new ActionRegistryArgs(serviceInstanceData.id) + registryArgs.getConf = yarn_site_config registryArgs.dest = outputDir describe registryArgs.toString() client.actionRegistry(registryArgs) - assert new File(outputDir, yarn_site_config + ".properties").exists() + assert new File(outputDir, yarn_site_config + ".xml").exists() - registryArgs.format = "xml" + registryArgs.format = "properties" client.actionRegistry(registryArgs) - assert new File(outputDir, yarn_site_config + ".xml").exists() + assert new File(outputDir, yarn_site_config + ".properties").exists() describe registryArgs.toString() Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy Fri May 16 14:51:27 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.ClusterDescription import org.apache.slider.api.ClusterNode +import org.apache.slider.api.RoleKeys import org.apache.slider.client.SliderClient import org.apache.slider.common.params.Arguments import org.apache.slider.common.tools.Duration @@ -259,7 +260,15 @@ class SliderTestUtils extends Assert { if (instanceCount != desiredCount) { roleCountFound = false; } - details.append("[$role]: desired: $desiredCount; actual: $instanceCount ") + details.append("[$role]: desired: $desiredCount;" + + " actual: $instanceCount ") + + // call out requested count, as this is a cause of problems on + // overloaded functional test clusters + def requested = status.roles[role][RoleKeys.ROLE_REQUESTED_INSTANCES] + if (requested != "0") { + details.append("requested: $requested ") + } } if (roleCountFound) { //successful @@ -277,8 +286,9 @@ class SliderTestUtils extends Assert { duration.finish(); describe("$operation: role count not met after $duration: $details") log.info(prettyPrint(status.toJsonString())) - fail( - "$operation: role counts not met after $duration: $details in \n$status ") + fail("$operation: role counts not met after $duration: " + + details.toString() + + " in \n$status ") } log.debug("Waiting: " + details) Thread.sleep(1000) Modified: incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy (original) +++ incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy Fri May 16 14:51:27 2014 @@ -49,7 +49,7 @@ import static org.apache.slider.common.S abstract class CommandTestBase extends SliderTestUtils { private static final Logger log = LoggerFactory.getLogger(CommandTestBase.class); - + public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP) public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP) public static final File SLIDER_BIN_DIRECTORY = new File( @@ -57,7 +57,8 @@ abstract class CommandTestBase extends S public static final File SLIDER_SCRIPT = new File( SLIDER_BIN_DIRECTORY, BIN_SLIDER).canonicalFile - public static final File SLIDER_CONF_DIRECTORY = new File(SLIDER_CONF_DIR).canonicalFile + public static final File SLIDER_CONF_DIRECTORY = new File( + SLIDER_CONF_DIR).canonicalFile public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY, CLIENT_CONFIG_FILENAME).canonicalFile @@ -65,16 +66,17 @@ abstract class CommandTestBase extends S public static final int THAW_WAIT_TIME public static final int FREEZE_WAIT_TIME - public static final int ACCUMULO_LAUNCH_WAIT_TIME public static final int SLIDER_TEST_TIMEOUT - public static final boolean ACCUMULO_TESTS_ENABLED public static final boolean FUNTESTS_ENABLED + public static final String YARN_RAM_REQUEST + + static { - SLIDER_CONFIG = new ConfLoader().loadSliderConf(SLIDER_CONF_XML); - THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, + SLIDER_CONFIG = new ConfLoader().loadSliderConf(SLIDER_CONF_XML); + THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, KEY_TEST_THAW_WAIT_TIME, 1000 * DEFAULT_THAW_WAIT_TIME_SECONDS) FREEZE_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, @@ -83,15 +85,14 @@ abstract class CommandTestBase extends S SLIDER_TEST_TIMEOUT = getTimeOptionMillis(SLIDER_CONFIG, KEY_TEST_TIMEOUT, 1000 * DEFAULT_TEST_TIMEOUT_SECONDS) - ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, - KEY_ACCUMULO_LAUNCH_TIME, - 1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS) FUNTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_SLIDER_FUNTESTS_ENABLED, true) - ACCUMULO_TESTS_ENABLED = - SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false) - } + YARN_RAM_REQUEST = SLIDER_CONFIG.get( + KEY_TEST_YARN_RAM_REQUEST, + DEFAULT_YARN_RAM_REQUEST) + + } @Rule public final Timeout testTimeout = new Timeout(SLIDER_TEST_TIMEOUT); @@ -110,9 +111,9 @@ abstract class CommandTestBase extends S SliderShell.script = SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") - + // now patch the settings with the path of the conf direcotry - + } /** @@ -143,7 +144,7 @@ abstract class CommandTestBase extends S } return property } - + /** * Exec any slider command * @param conf @@ -217,7 +218,7 @@ abstract class CommandTestBase extends S ACTION_FREEZE, name ]) } - + static SliderShell freeze(String name, Collection<String> args) { slider([ACTION_FREEZE, name] + args) } @@ -234,18 +235,18 @@ abstract class CommandTestBase extends S static SliderShell getConf(int result, String name) { slider(result, - [ - ACTION_GETCONF, name - ]) + [ + ACTION_GETCONF, name + ]) } - + static SliderShell killContainer(String name, String containerID) { slider(0, - [ - ACTION_KILL_CONTAINER, - name, - containerID - ]) + [ + ACTION_KILL_CONTAINER, + name, + containerID + ]) } static SliderShell list(String name) { @@ -276,9 +277,9 @@ abstract class CommandTestBase extends S static SliderShell status(int result, String name) { slider(result, - [ - ACTION_STATUS, name - ]) + [ + ACTION_STATUS, name + ]) } static SliderShell thaw(String name) { @@ -289,23 +290,24 @@ abstract class CommandTestBase extends S static SliderShell thaw(int result, String name) { slider(result, - [ - ACTION_THAW, name - ]) + [ + ACTION_THAW, name + ]) } static SliderShell thaw(String name, Collection<String> args) { slider([ACTION_THAW, name] + args) } - + static SliderShell registry(int result, Collection<String> commands) { slider(result, - [ ACTION_REGISTRY ] + commands + [ACTION_REGISTRY] + commands ) } + static SliderShell registry(Collection<String> commands) { slider(0, - [ ACTION_REGISTRY ] + commands + [ACTION_REGISTRY] + commands ) } @@ -325,7 +327,7 @@ abstract class CommandTestBase extends S /** * If the functional tests are enabled, set up the cluster - * + * * @param cluster */ static void setupCluster(String cluster) { @@ -419,8 +421,8 @@ abstract class CommandTestBase extends S List<String> argsList = [action, clustername] argsList << ARG_ZKHOSTS << - SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM) - + SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM) + if (blockUntilRunning) { argsList << ARG_WAIT << Integer.toString(THAW_WAIT_TIME) @@ -470,13 +472,15 @@ abstract class CommandTestBase extends S } public Path buildClusterPath(String clustername) { - return new Path(clusterFS.homeDirectory, "${SliderKeys.SLIDER_BASE_DIRECTORY}/cluster/${clustername}") + return new Path( + clusterFS.homeDirectory, + "${SliderKeys.SLIDER_BASE_DIRECTORY}/cluster/${clustername}") } public ClusterDescription killAmAndWaitForRestart( SliderClient sliderClient, String cluster) { - + assert cluster slider(0, [ ACTION_AM_SUICIDE, cluster, @@ -487,8 +491,8 @@ abstract class CommandTestBase extends S - def sleeptime = SLIDER_CONFIG.getInt( KEY_AM_RESTART_SLEEP_TIME, - DEFAULT_AM_RESTART_SLEEP_TIME) + def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME, + DEFAULT_AM_RESTART_SLEEP_TIME) sleep(sleeptime) ClusterDescription status @@ -515,9 +519,4 @@ abstract class CommandTestBase extends S assume(FUNTESTS_ENABLED, "Functional tests disabled") } - public static void assumeAccumuloTestsEnabled() { - assumeFunctionalTestsEnabled() - assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled") - } - } Modified: incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy (original) +++ incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy Fri May 16 14:51:27 2014 @@ -146,7 +146,10 @@ class SliderShell extends Shell { assert shell != null if (shell.ret != errorCode) { shell.dump() - throw new SliderException(shell.ret,"Expected exit code %d - actual=%d", errorCode, shell.ret) + throw new SliderException(shell.ret, + "Expected exit code of command %s : %d - actual=%d", + shell.command, + errorCode, shell.ret) } return errorCode } Modified: incubator/slider/trunk/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy (original) +++ incubator/slider/trunk/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy Fri May 16 14:51:27 2014 @@ -21,6 +21,9 @@ package org.apache.slider.providers.accu import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_APPCONF import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_TAR import static org.apache.slider.api.ResourceKeys.YARN_MEMORY +import static org.apache.slider.common.SliderXMLConfKeysForTesting.DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS +import static org.apache.slider.common.SliderXMLConfKeysForTesting.KEY_ACCUMULO_LAUNCH_TIME +import static org.apache.slider.common.SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_ENABLED import static org.apache.slider.providers.accumulo.AccumuloKeys.* import static org.apache.slider.common.params.Arguments.ARG_PROVIDER import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT @@ -46,6 +49,23 @@ import org.junit.Before */ abstract class AccumuloCommandTestBase extends CommandTestBase { + public static final int ACCUMULO_LAUNCH_WAIT_TIME + public static final boolean ACCUMULO_TESTS_ENABLED + + static { + ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, + KEY_ACCUMULO_LAUNCH_TIME, + 1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS) + ACCUMULO_TESTS_ENABLED = + SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false) + } + + + public static void assumeAccumuloTestsEnabled() { + assumeFunctionalTestsEnabled() + assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled") + } + @Before public void verifyPreconditions() { Modified: incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy (original) +++ incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy Fri May 16 14:51:27 2014 @@ -18,16 +18,20 @@ package org.apache.slider.providers.hbase.funtest +import org.apache.slider.api.ResourceKeys +import org.apache.slider.common.params.Arguments import org.apache.slider.funtest.categories.FunctionalTests import org.apache.slider.funtest.framework.CommandTestBase import org.apache.slider.funtest.framework.SliderShell -import org.apache.slider.common.params.Arguments import org.apache.slider.providers.hbase.HBaseClientProvider import org.apache.slider.providers.hbase.HBaseKeys import org.junit.Before import org.junit.BeforeClass import static org.apache.slider.common.SliderXMLConfKeysForTesting.* +import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT +import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_MASTER +import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER /** * Anything specific to HBase tests @@ -41,9 +45,9 @@ abstract class HBaseCommandTestBase exte static { HBASE_TESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_HBASE_ENABLED, true) - HBASE_LAUNCH_WAIT_TIME = SLIDER_CONFIG.getInt( - KEY_TEST_HBASE_LAUNCH_TIME, - DEFAULT_HBASE_LAUNCH_TIME_SECONDS) + HBASE_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, + KEY_TEST_HBASE_LAUNCH_TIME, + 1000 * DEFAULT_HBASE_LAUNCH_TIME_SECONDS) } @BeforeClass @@ -92,6 +96,11 @@ abstract class HBaseCommandTestBase exte SLIDER_CONFIG.getTrimmed(KEY_TEST_HBASE_APPCONF) argsList << Arguments.ARG_PROVIDER << HBaseKeys.PROVIDER_HBASE + // add the RAM requirements + argsList << ARG_RES_COMP_OPT << ROLE_MASTER << + ResourceKeys.YARN_MEMORY << YARN_RAM_REQUEST + argsList << ARG_RES_COMP_OPT << ROLE_WORKER << + ResourceKeys.YARN_MEMORY << YARN_RAM_REQUEST SliderShell shell = createSliderApplication( name, Modified: incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy (original) +++ incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy Fri May 16 14:51:27 2014 @@ -21,20 +21,25 @@ package org.apache.slider.providers.hbas import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.api.ClusterDescription +import org.apache.slider.api.RoleKeys +import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys -import org.apache.slider.api.ClusterDescription -import org.apache.slider.api.RoleKeys +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.tools.ConfigHelper import org.apache.slider.core.registry.info.RegistryNaming import org.apache.slider.funtest.framework.FuntestProperties -import org.apache.slider.common.tools.ConfigHelper -import org.apache.slider.common.params.Arguments -import org.apache.slider.client.SliderClient import org.apache.slider.providers.hbase.HBaseConfigFileOptions import org.apache.slider.providers.hbase.HBaseTestUtils import org.apache.slider.server.appmaster.PublishedArtifacts -import org.apache.zookeeper.* +import org.apache.zookeeper.KeeperException +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZKUtil +import org.apache.zookeeper.ZooKeeper import org.junit.After import org.junit.Before import org.junit.Test @@ -120,8 +125,10 @@ public class TestFunctionalHBaseCluster Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(sliderClient) HBaseTestUtils.assertHBaseMasterFound(clientConf) - HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient, clusterName, - numWorkers, HBASE_LAUNCH_WAIT_TIME) + HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient, + clusterName, + numWorkers, + HBASE_LAUNCH_WAIT_TIME) clusterOperations( clusterName, @@ -172,8 +179,6 @@ public class TestFunctionalHBaseCluster //unknown service type registry(EXIT_NOT_FOUND, [ARG_LIST, ARG_SERVICETYPE, "org.apache.something"]) - registry(EXIT_NOT_FOUND, - [ARG_LIST, ARG_SERVICETYPE, ""]) registry(EXIT_NOT_FOUND, [ARG_LIST, ARG_NAME, "cluster-with-no-name"]) @@ -190,16 +195,29 @@ public class TestFunctionalHBaseCluster registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, "unknown"]) registry([ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG, ARG_NAME, name]) - registry([ARG_GETCONF, "no-such-config", + registry(EXIT_NOT_FOUND, [ARG_GETCONF, "no-such-config", ARG_NAME, name]) - registry([ARG_GETCONF, "illegal config name!", + registry(EXIT_NOT_FOUND, [ARG_GETCONF, "illegal/config/name!", ARG_NAME, name]) registry(EXIT_NOT_FOUND,[ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG, ARG_NAME, name, ARG_INTERNAL]) + def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG + registry([ARG_GETCONF, yarn_site_config, + ARG_NAME, name]) + + File getConfDir = new File("target/$clusterName/getconf") + getConfDir.mkdirs(); + registry([ARG_GETCONF, yarn_site_config, + ARG_NAME, name, + ARG_DEST, getConfDir.absolutePath]) + File retrieved = new File(getConfDir, yarn_site_config +".xml") + def confFromFile = ConfigHelper.loadConfFromFile(retrieved) + assert confFromFile.get(YarnConfiguration.RM_ADDRESS) + } } Modified: incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy (original) +++ incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy Fri May 16 14:51:27 2014 @@ -21,13 +21,12 @@ package org.apache.slider.providers.hbas import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.fs.Path -import org.apache.slider.core.main.LauncherExitCodes import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys -import org.apache.slider.funtest.framework.CommandTestBase -import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.main.LauncherExitCodes +import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.providers.hbase.HBaseKeys import org.junit.AfterClass import org.junit.BeforeClass Modified: incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy (original) +++ incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy Fri May 16 14:51:27 2014 @@ -20,14 +20,14 @@ package org.apache.slider.providers.hbas import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.slider.common.SliderExitCodes -import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.api.ClusterDescription import org.apache.slider.api.StatusKeys -import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions -import org.apache.slider.client.SliderClient +import org.apache.slider.funtest.framework.FuntestProperties import org.junit.After import org.junit.Before import org.junit.Test Modified: incubator/slider/trunk/src/site/markdown/developing/functional_tests.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/functional_tests.md?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/developing/functional_tests.md (original) +++ incubator/slider/trunk/src/site/markdown/developing/functional_tests.md Fri May 16 14:51:27 2014 @@ -24,7 +24,7 @@ # Functional Tests -The functional test suite is designed to run the executables against +The functional test suite is designed to test slider against a live cluster. For these to work you need @@ -112,6 +112,12 @@ need to be changed <value>180000</value> </property> + <property> + <name>slider.test.yarn.ram</name> + <description>Size in MB to ask for containers</description> + <value>192</value> + </property> + Note that while the same properties need to be set in `slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local @@ -215,7 +221,7 @@ Optional parameters: <property> <name>slider.test.hbase.launch.wait.seconds</name> <description>Time to wait in seconds for HBase to start</description> - <value>180000</value> + <value>1800</value> </property> #### Accumulo configuration options @@ -233,7 +239,7 @@ Optional parameters <property> <name>slider.test.accumulo.launch.wait.seconds</name> <description>Time to wait in seconds for Accumulo to start</description> - <value>180000</value> + <value>1800</value> </property> ### Configuring the YARN cluster for tests @@ -360,9 +366,9 @@ The functional tests all mvn install -DskipTests 1. Start the YARN cluster/set up proxies to connect to it, etc. -1. In the `slider-funtest` dir, run the test +1. In the `slider-funtest` dir, run the tests - mvn test -Dtest=TestHBaseCreateCluster + mvn test A common mistake during development is to rebuild the `slider-core` JARs then the `slider-funtest` tests without rebuilding the `slider-assembly`. @@ -383,6 +389,10 @@ testing, you must build/install all the ## Troubleshooting the functional tests +1. If application instances fail to come up as there are still outstanding +requests, it means that YARN didn't have the RAM/cores to spare for the number +of containers. Edit the `slider.test.yarn.ram` to make it smaller. + 1. If you are testing in a local VM and stops responding, it'll have been swapped out to RAM. Rebooting can help, but for a long term fix go through all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to Modified: incubator/slider/trunk/src/site/markdown/manpage.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/manpage.md?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/manpage.md (original) +++ incubator/slider/trunk/src/site/markdown/manpage.md Fri May 16 14:51:27 2014 @@ -387,7 +387,7 @@ List the configurations exported by of a #### `registry --getconf <configuration> [--format (xml|json|properties)] [--dest <path>] [--internal] ` get the configuration -Get a named configuration +Get a named configuration in a chosen format. Default: XML `--dest <path>` : the filename or directory to save a configuration to. `--format (xml|json|properties)` defines the output format Modified: incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml?rev=1595221&r1=1595220&r2=1595221&view=diff ============================================================================== --- incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml (original) +++ incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml Fri May 16 14:51:27 2014 @@ -31,7 +31,7 @@ <property> <name>fs.defaultFS</name> - <value>hdfs://sandbox:8020</value> + <value>hdfs://sandbox.hortonworks.com:8020</value> </property> <property> @@ -67,7 +67,7 @@ <property> <name>slider.test.hbase.tar</name> <description>Path to the HBase Tar file in HDFS</description> - <value>hdfs://sandbox:8020/hbase-0.98.1-bin.tar.gz</value> + <value>${fs.defaultFS}/hbase-0.98.1-bin.tar.gz</value> </property> @@ -88,7 +88,7 @@ <property> <name>slider.test.accumulo.tar</name> <description>Path to the Accumulo Tar file in HDFS</description> - <value>hdfs://sandbox:8020/user/slider/accumulo.tar</value> + <value>${fs.defaultFS}/user/slider/accumulo.tar</value> </property>