Author: jmaron Date: Mon May 12 18:30:30 2014 New Revision: 1594045 URL: http://svn.apache.org/r1594045 Log: SLIDER-43 validate resource inputs start with yarn and app conf components do not
Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java Modified: incubator/slider/trunk/app-packages/accumulo-v1_5/resources.json incubator/slider/trunk/app-packages/hbase-v0_96/resources.json incubator/slider/trunk/app-packages/storm-v0_91/resources.json incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java incubator/slider/trunk/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json incubator/slider/trunk/slider-core/src/test/app_packages/test_command_log/resources.json incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json incubator/slider/trunk/src/site/markdown/getting_started.md incubator/slider/trunk/src/site/markdown/slider_specs/creating_app_definitions.md incubator/slider/trunk/src/site/markdown/slider_specs/resource_specification.md incubator/slider/trunk/src/site/markdown/specification/cli-actions.md Modified: incubator/slider/trunk/app-packages/accumulo-v1_5/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/app-packages/accumulo-v1_5/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/app-packages/accumulo-v1_5/resources.json (original) +++ incubator/slider/trunk/app-packages/accumulo-v1_5/resources.json Mon May 12 18:30:30 2014 @@ -6,26 +6,26 @@ }, "components": { "ACCUMULO_MASTER": { - "role.priority": "1", - "component.instances": "1" + "yarn.role.priority": "1", + "yarn.component.instances": "1" }, "slider-appmaster": { }, "ACCUMULO_TSERVER": { - "role.priority": "2", - "component.instances": "1" + "yarn.role.priority": "2", + "yarn.component.instances": "1" }, "ACCUMULO_MONITOR": { - "role.priority": "3", - "component.instances": "1" + "yarn.role.priority": "3", + "yarn.component.instances": "1" }, "ACCUMULO_GC": { - "role.priority": "4", - "component.instances": "1" + "yarn.role.priority": "4", + "yarn.component.instances": "1" }, "ACCUMULO_TRACER": { - "role.priority": "5", - "component.instances": "1" + "yarn.role.priority": "5", + "yarn.component.instances": "1" } } } Modified: incubator/slider/trunk/app-packages/hbase-v0_96/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/app-packages/hbase-v0_96/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/app-packages/hbase-v0_96/resources.json (original) +++ incubator/slider/trunk/app-packages/hbase-v0_96/resources.json Mon May 12 18:30:30 2014 @@ -6,14 +6,14 @@ }, "components": { "HBASE_MASTER": { - "role.priority": "1", - "component.instances": "1" + "yarn.role.priority": "1", + "yarn.component.instances": "1" }, "slider-appmaster": { }, "HBASE_REGIONSERVER": { - "role.priority": "2", - "component.instances": "1" + "yarn.role.priority": "2", + "yarn.omponent.instances": "1" } } } \ No newline at end of file Modified: incubator/slider/trunk/app-packages/storm-v0_91/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/app-packages/storm-v0_91/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/app-packages/storm-v0_91/resources.json (original) +++ incubator/slider/trunk/app-packages/storm-v0_91/resources.json Mon May 12 18:30:30 2014 @@ -8,24 +8,24 @@ "slider-appmaster": { }, "NIMBUS": { - "role.priority": "1", - "component.instances": "1" + "yarn.role.priority": "1", + "yarn.component.instances": "1" }, "STORM_REST_API": { - "role.priority": "2", - "component.instances": "1" + "yarn.role.priority": "2", + "yarn.component.instances": "1" }, "STORM_UI_SERVER": { - "role.priority": "3", - "component.instances": "1" + "yarn.role.priority": "3", + "yarn.component.instances": "1" }, "DRPC_SERVER": { - "role.priority": "4", - "component.instances": "1" + "yarn.role.priority": "4", + "yarn.component.instances": "1" }, "SUPERVISOR": { - "role.priority": "5", - "component.instances": "1" + "yarn.role.priority": "5", + "yarn.component.instances": "1" } } } \ No newline at end of file Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java Mon May 12 18:30:30 2014 @@ -28,7 +28,7 @@ public interface ResourceKeys { * #of instances of a component * */ - String COMPONENT_INSTANCES = "component.instances"; + String COMPONENT_INSTANCES = "yarn.component.instances"; /** * Amount of memory to ask YARN for in MB. @@ -63,10 +63,10 @@ public interface ResourceKeys { * 2. this must be >= 1 * 3. this must not match any other role priority in the cluster. */ - String COMPONENT_PRIORITY = "role.priority"; + String COMPONENT_PRIORITY = "yarn.role.priority"; /** * placement policy */ - String COMPONENT_PLACEMENT_POLICY = "component.placement.policy"; + String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy"; } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java Mon May 12 18:30:30 2014 @@ -67,6 +67,8 @@ import org.apache.slider.core.conf.Aggre import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.conf.ResourcesInputPropertiesValidator; +import org.apache.slider.core.conf.TemplateInputPropertiesValidator; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; @@ -396,7 +398,8 @@ public class SliderClient extends Abstra //load in any specified on the command line if (buildInfo.resources != null) { try { - resources.mergeFile(buildInfo.resources); + resources.mergeFile(buildInfo.resources, + new ResourcesInputPropertiesValidator()); } catch (IOException e) { throw new BadConfigException(e, @@ -408,7 +411,8 @@ public class SliderClient extends Abstra } if (buildInfo.template != null) { try { - appConf.mergeFile(buildInfo.template); + appConf.mergeFile(buildInfo.template, + new TemplateInputPropertiesValidator()); } catch (IOException e) { throw new BadConfigException(e, "incorrect argument to %s: \"%s\" : %s ", @@ -1097,7 +1101,6 @@ public class SliderClient extends Abstra /** * Build an exit code for an application Id and its report. * If the report parameter is null, the app is killed - * @param appId app * @param report report * @return the exit code */ @@ -2071,8 +2074,6 @@ public class SliderClient extends Abstra /** * Look up an instance - * @param id instance ID - * @param serviceType service type * @return instance data * @throws UnknownApplicationInstanceException no match * @throws SliderException other failures Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java?rev=1594045&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java (added) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java Mon May 12 18:30:30 2014 @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.conf; + +import org.apache.slider.core.exceptions.BadConfigException; + +/** + * + */ +public abstract class AbstractInputPropertiesValidator implements InputPropertiesValidator{ + + abstract void validatePropertyNamePrefix(String key) throws BadConfigException; + + public void validate(ConfTreeOperations props) + throws BadConfigException { + validateGlobalProperties(props); + validateComponentProperties(props); + + } + + protected void validateComponentProperties(ConfTreeOperations props) + throws BadConfigException { + for (String compName : props.getComponentNames()) { + for (String key : props.getComponent(compName).keySet()) { + validatePropertyNamePrefix(key); + } + } + } + + abstract void validateGlobalProperties(ConfTreeOperations props) + throws BadConfigException; + +} Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java Mon May 12 18:30:30 2014 @@ -64,15 +64,27 @@ public class ConfTreeOperations { * @throws BadConfigException */ public void validate() throws BadConfigException { + validate(null); + } + + /** + * Validate the configuration + * @param validator a provided properties validator + * @throws BadConfigException + */ + public void validate(InputPropertiesValidator validator) throws BadConfigException { String version = confTree.schema; if (version == null) { throw new BadConfigException("'version' undefined"); } if (!PersistKeys.SCHEMA.equals(version)) { throw new BadConfigException( - "version %s incompatible with supported version %s", - version, - PersistKeys.SCHEMA); + "version %s incompatible with supported version %s", + version, + PersistKeys.SCHEMA); + } + if (validator != null) { + validator.validate(this); } } @@ -335,10 +347,21 @@ public class ConfTreeOperations { * @throws BadConfigException if the file is invalid */ public void mergeFile(File file) throws IOException, BadConfigException { + mergeFile(file, null); + } + + /** + * Load from a file and merge it in + * @param file file + * @param validator properties validator + * @throws IOException any IO problem + * @throws BadConfigException if the file is invalid + */ + public void mergeFile(File file, InputPropertiesValidator validator) throws IOException, BadConfigException { ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); ConfTree tree = confTreeSerDeser.fromFile(file); ConfTreeOperations ops = new ConfTreeOperations(tree); - ops.validate(); + ops.validate(validator); merge(ops.confTree); } Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java?rev=1594045&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java (added) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java Mon May 12 18:30:30 2014 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.conf; + +import org.apache.slider.core.exceptions.BadConfigException; + +/** + * + */ +public interface InputPropertiesValidator { + void validate(ConfTreeOperations props) throws BadConfigException; +} Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java?rev=1594045&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java (added) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java Mon May 12 18:30:30 2014 @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.conf; + +import org.apache.slider.core.exceptions.BadConfigException; + +/** + * + */ +public class ResourcesInputPropertiesValidator + extends AbstractInputPropertiesValidator { + + void validatePropertyNamePrefix(String key) throws BadConfigException { + if (!key.startsWith("yarn.")) { + throw new BadConfigException( + "argument %s does not have 'yarn.' prefix", key); + } + } + + protected void validateGlobalProperties(ConfTreeOperations props) + throws BadConfigException { + for (String key : props.getGlobalOptions().keySet()) { + validatePropertyNamePrefix(key); + } + } +} Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java?rev=1594045&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java (added) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java Mon May 12 18:30:30 2014 @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.conf; + +import org.apache.slider.core.exceptions.BadConfigException; + +/** + * + */ +public class TemplateInputPropertiesValidator + extends AbstractInputPropertiesValidator { + + void validatePropertyNamePrefix(String key) throws BadConfigException { + if (key.startsWith("yarn.")) { + throw new BadConfigException( + "argument %s does not have 'yarn.' prefix", key); + } + } + + @Override + void validateGlobalProperties(ConfTreeOperations props) { + // do nothing + } +} Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java Mon May 12 18:30:30 2014 @@ -31,7 +31,6 @@ import org.apache.slider.server.appmaste import org.apache.slider.server.appmaster.web.rest.AMWebServices; import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider; import org.apache.slider.server.services.curator.CuratorHelper; -import org.apache.slider.server.services.curator.RegistryBinderService; import org.apache.slider.server.services.registry.RegistryDiscoveryContext; import org.apache.slider.server.services.registry.RegistryRestResources; import org.apache.slider.server.services.registry.SliderRegistryService; Modified: incubator/slider/trunk/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json (original) +++ incubator/slider/trunk/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json Mon May 12 18:30:30 2014 @@ -10,7 +10,7 @@ "components": { "slider-appmaster": { - "component.instances": "1", + "yarn.component.instances": "1", "yarn.vcores": "1", "yarn.memory": "256" } Modified: incubator/slider/trunk/slider-core/src/test/app_packages/test_command_log/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/app_packages/test_command_log/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/app_packages/test_command_log/resources.json (original) +++ incubator/slider/trunk/slider-core/src/test/app_packages/test_command_log/resources.json Mon May 12 18:30:30 2014 @@ -6,8 +6,8 @@ }, "components": { "COMMAND_LOGGER": { - "role.priority": "1", - "component.instances": "1" + "yarn.role.priority": "1", + "yarn.component.instances": "1" }, "slider-appmaster": { } Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy Mon May 12 18:30:30 2014 @@ -44,7 +44,7 @@ public abstract class AgentMiniClusterTe public static void createSubConfFiles() { File destDir = new File("target/agent_minicluster_testbase") destDir.mkdirs() - agentConf = new File(destDir, "agentconf") + agentConf = new File(destDir, "agentconf.zip") agentConf.createNewFile() agentDef = new File(destDir, "agentdef") agentDef.createNewFile() Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy Mon May 12 18:30:30 2014 @@ -18,6 +18,7 @@ package org.apache.slider.core.conf +import org.apache.slider.core.exceptions.BadConfigException import org.apache.slider.core.persist.JsonSerDeser import org.junit.Assert import org.junit.Test @@ -53,4 +54,31 @@ class TestConfTreeLoadExamples extends A ops.validate() } + + @Test + public void testLoadResourceWithValidator() throws Throwable { + def confTree = confTreeJsonSerDeser.fromResource(resource) + ConfTreeOperations ops = new ConfTreeOperations(confTree) + ops.resolve() + if (resource.endsWith("resources.json")) { + // these should pass since they are configured conrrectly with "yarn." + // properties + ops.validate(new ResourcesInputPropertiesValidator()) + } else if (resource.startsWith("app_configuration")) { + ops.validate(new TemplateInputPropertiesValidator()) + } + else { + // these have properties with other prefixes so they should generate + // BadConfigExceptions + try { + ops.validate(new ResourcesInputPropertiesValidator()) + if ( !resource.endsWith(ExampleConfResources.empty)) { + fail (resource + " should have generated validation exception") + } + } catch (BadConfigException e) { + // ignore + } + + } + } } Modified: incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json (original) +++ incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json Mon May 12 18:30:30 2014 @@ -12,14 +12,14 @@ "components": { "master": { - "instances": "1", + "yarn.component.instances": "1", "yarn.vcores": "1", "yarn.memory": "1024" }, "worker": { - "instances":"5", + "yarn.component.instances":"5", "yarn.vcores": "1", "yarn.memory": "512" } } -} \ No newline at end of file +} Modified: incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json (original) +++ incubator/slider/trunk/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json Mon May 12 18:30:30 2014 @@ -6,8 +6,8 @@ "components": { "node": { "yarn.memory": "256", - "component.instances": "5", - "role.priority":"1" + "yarn.component.instances": "5", + "yarn.role.priority":"1" } } } Modified: incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json (original) +++ incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json Mon May 12 18:30:30 2014 @@ -12,19 +12,19 @@ "components": { "master": { - "component.instances": "1" + "yarn.component.instances": "1" }, "tserver": { - "component.instances": "2" + "yarn.component.instances": "2" }, "monitor": { - "component.instances": "0" + "yarn.component.instances": "0" }, "tracer": { - "component.instances": "0" + "yarn.component.instances": "0" }, "gc": { - "component.instances": "0" + "yarn.component.instances": "0" } } } \ No newline at end of file Modified: incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json (original) +++ incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json Mon May 12 18:30:30 2014 @@ -11,12 +11,12 @@ "components": { "master": { - "component.instances": "1", + "yarn.component.instances": "1", "yarn.vcores": "1", "yarn.memory": "1024" }, "worker": { - "component.instances": "2", + "yarn.component.instances": "2", "yarn.vcores": "1", "yarn.memory": "768" } Modified: incubator/slider/trunk/src/site/markdown/getting_started.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/getting_started.md?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/getting_started.md (original) +++ incubator/slider/trunk/src/site/markdown/getting_started.md Mon May 12 18:30:30 2014 @@ -377,25 +377,24 @@ Once started, applications can be frozen "slider-appmaster" : { }, "NIMBUS" : { - "role.priority" : "1", - "component.instances" : "1" + "yarn.role.priority" : "1", + "yarn.component.instances" : "1" }, "STORM_REST_API" : { - "wait.heartbeat" : "3", - "role.priority" : "2", - "component.instances" : "1" + "yarn.role.priority" : "2", + "yarn.component.instances" : "1" }, "STORM_UI_SERVER" : { - "role.priority" : "3", - "component.instances" : "1" + "yarn.role.priority" : "3", + "yarn.component.instances" : "1" }, "DRPC_SERVER" : { - "role.priority" : "4", - "component.instances" : "1" + "yarn.role.priority" : "4", + "yarn.component.instances" : "1" }, "SUPERVISOR" : { - "role.priority" : "5", - "component.instances" : "1" + "yarn.role.priority" : "5", + "yarn.component.instances" : "1" } } } @@ -435,19 +434,14 @@ Once started, applications can be frozen }, "components" : { "HBASE_MASTER" : { - "wait.heartbeat" : "5", - "role.priority" : "1", - "component.instances" : "1", - "role.script" : "scripts/hbase_master.py" + "yarn.role.priority" : "1", + "yarn.component.instances" : "1" }, "slider-appmaster" : { - "jvm.heapsize" : "256M" }, "HBASE_REGIONSERVER" : { - "wait.heartbeat" : "3", - "role.priority" : "2", - "component.instances" : "1", - "role.script" : "scripts/hbase_regionserver.py" + "yarn.role.priority" : "2", + "yarn.component.instances" : "1" } } } Modified: incubator/slider/trunk/src/site/markdown/slider_specs/creating_app_definitions.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/slider_specs/creating_app_definitions.md?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/slider_specs/creating_app_definitions.md (original) +++ incubator/slider/trunk/src/site/markdown/slider_specs/creating_app_definitions.md Mon May 12 18:30:30 2014 @@ -122,19 +122,15 @@ Sample: }, "components" : { "HBASE_MASTER" : { - "wait.heartbeat" : "5", - "role.priority" : "1", - "component.instances" : "1", - "role.script" : "scripts/hbase_master.py" + "yarn.role.priority" : "1", + "yarn.component.instances" : "1" }, "slider-appmaster" : { "jvm.heapsize" : "256M" }, "HBASE_REGIONSERVER" : { - "wait.heartbeat" : "3", - "role.priority" : "2", - "component.instances" : "1", - "role.script" : "scripts/hbase_regionserver.py" + "yarn.role.priority" : "2", + "yarn.component.instances" : "1" } } } Modified: incubator/slider/trunk/src/site/markdown/slider_specs/resource_specification.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/slider_specs/resource_specification.md?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/slider_specs/resource_specification.md (original) +++ incubator/slider/trunk/src/site/markdown/slider_specs/resource_specification.md Mon May 12 18:30:30 2014 @@ -31,23 +31,20 @@ An example resource requirement for an a "components" : { "worker" : { "yarn.memory" : "768", - "env.MALLOC_ARENA_MAX" : "4", - "component.instances" : "1", - "component.name" : "worker", + "yarn.component.instances" : "1", + "yarn.component.name" : "worker", "yarn.vcores" : "1" }, "slider" : { "yarn.memory" : "256", - "env.MALLOC_ARENA_MAX" : "4", - "component.instances" : "1", - "component.name" : "slider", + "yarn.component.instances" : "1", + "yarn.component.name" : "slider", "yarn.vcores" : "1" }, "master" : { "yarn.memory" : "1024", - "env.MALLOC_ARENA_MAX" : "4", - "component.instances" : "1", - "component.name" : "master", + "yarn.component.instances" : "1", + "yarn.component.name" : "master", "yarn.vcores" : "1" } } Modified: incubator/slider/trunk/src/site/markdown/specification/cli-actions.md URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/specification/cli-actions.md?rev=1594045&r1=1594044&r2=1594045&view=diff ============================================================================== --- incubator/slider/trunk/src/site/markdown/specification/cli-actions.md (original) +++ incubator/slider/trunk/src/site/markdown/specification/cli-actions.md Mon May 12 18:30:30 2014 @@ -350,7 +350,7 @@ which release requests have been made mu If the internal state of the Slider AM is defined as `AppState` forall r in clusterspec.roles : - r["component.instances"] == + r["yarn.component.instances"] == AppState.Roles[r].live + AppState.Roles[r].requested - AppState.Roles[r].released The `AppState` represents Slider's view of the external YARN system state, based on its @@ -360,7 +360,7 @@ It is indirectly observable from the clu forall r in AM.getJSONClusterStatus().roles : - r["component.instances"] == + r["yarn.component.instances"] == r["role.actual.instances"] + r["role.requested.instances"] - r["role.releasing.instances"] Slider does not consider it an error if the number of actual instances remains below @@ -473,7 +473,7 @@ which will change the desired steady-sta let updatedSpec = originalspec where: forall (name, size) in components : - updatedSpec.roles[name]["component.instances"] == size + updatedSpec.roles[name]["yarn.component.instances"] == size data(HDFS', cluster-json-path(HDFS', instancename)) == updatedSpec rpc-connection(slider-live-instances(YARN(t2))[0], SliderClusterProtocol) let flexed = rpc-connection(slider-live-instances(YARN(t2))[0], SliderClusterProtocol).flexClusterupdatedSpec) @@ -496,7 +496,7 @@ and be `True` iff the desired steady sta #### Postconditions forall role in AppState.Roles.keys: - AppState'.Roles'[role].desiredCount = updatedSpec[roles]["component.instances"] + AppState'.Roles'[role].desiredCount = updatedSpec[roles]["yarn.component.instances"] result = AppState' != AppState