SLIDER-324. Exports should be attributed to individual master components rather than only one master component
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc9561d5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc9561d5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc9561d5 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: bc9561d59d3c0a68a6abaa47e8a91b87a36a1287 Parents: 599e972 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Fri Aug 15 13:32:12 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Fri Aug 15 13:32:12 2014 -0700 ---------------------------------------------------------------------- app-packages/storm/metainfo.xml | 28 ++++---- .../agent/application/metadata/Component.java | 21 ++++-- .../application/metadata/ComponentExport.java | 54 ++++++++++++++ .../application/metadata/MetainfoParser.java | 9 +-- .../agent/TestAgentProviderService.java | 23 +++--- .../agent/application/metadata/metainfo.xml | 74 ++++++++++++++++++-- 6 files changed, 171 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index e94bc62..d9004ad 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -29,21 +29,25 @@ <name>QuickLinks</name> <exports> <export> - <name>org.apache.slider.jmx</name> + <name>app.jmx</name> <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> </export> <export> - <name>org.apache.slider.monitor</name> + <name>app.monitor</name> <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> </export> <export> - <name>org.apache.slider.metrics</name> + <name>app.metrics</name> <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> </export> <export> - <name>org.apache.slider.ganglia</name> + <name>ganglia.ui</name> <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> </export> + <export> + <name>nimbus.url</name> + <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value> + </export> </exports> </exportGroup> </exportGroups> @@ -78,6 +82,7 @@ <name>NIMBUS</name> <category>MASTER</category> <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports> <commandScript> <script>scripts/nimbus.py</script> <scriptType>PYTHON</scriptType> @@ -89,6 +94,7 @@ <name>STORM_REST_API</name> <category>MASTER</category> <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-app.jmx</appExports> <commandScript> <script>scripts/rest_api.py</script> <scriptType>PYTHON</scriptType> @@ -100,12 +106,12 @@ <name>SUPERVISOR</name> <category>SLAVE</category> <autoStartOnFailure>true</autoStartOnFailure> - <exports> - <export> + <componentExports> + <componentExport> <name>log_viewer_port</name> <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value> - </export> - </exports> + </componentExport> + </componentExports> <commandScript> <script>scripts/supervisor.py</script> <scriptType>PYTHON</scriptType> @@ -117,6 +123,7 @@ <name>STORM_UI_SERVER</name> <category>MASTER</category> <publishConfig>true</publishConfig> + <appExports>QuickLinks-app.monitor</appExports> <autoStartOnFailure>true</autoStartOnFailure> <commandScript> <script>scripts/ui_server.py</script> @@ -148,10 +155,5 @@ </packages> </osSpecific> </osSpecifics> - - <configuration-dependencies> - <config-type>storm-site</config-type> - <config-type>global</config-type> - </configuration-dependencies> </application> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index acf6ecb..03c64d4 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -29,12 +29,13 @@ public class Component { String minInstanceCount; String maxInstanceCount; String autoStartOnFailure; + String appExports; CommandScript commandScript; - List<Export> exports; + List<ComponentExport> componentExports; public Component() { publishConfig = Boolean.FALSE.toString(); - exports = new ArrayList<Export>(); + componentExports = new ArrayList<ComponentExport>(); } public String getName() { @@ -69,6 +70,14 @@ public class Component { this.autoStartOnFailure = autoStartOnFailure; } + public String getAppExports() { + return appExports; + } + + public void setAppExports(String appExports) { + this.appExports = appExports; + } + public String getMinInstanceCount() { return minInstanceCount; } @@ -93,12 +102,12 @@ public class Component { this.commandScript = commandScript; } - public void addExport(Export export) { - exports.add(export); + public void addComponentExport(ComponentExport export) { + componentExports.add(export); } - public List<Export> getExports() { - return exports; + public List<ComponentExport> getComponentExports() { + return componentExports; } public Boolean getRequiresAutoRestart() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java new file mode 100644 index 0000000..a18854c --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +/** + * + */ +public class ComponentExport { + String name; + String value; + + public ComponentExport() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"value\": ").append(value); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java index ada5b83..4b91ff7 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -60,10 +60,11 @@ public class MetainfoParser { digester.addBeanPropertySetter("*/component/minInstanceCount"); digester.addBeanPropertySetter("*/component/maxInstanceCount"); digester.addBeanPropertySetter("*/component/autoStartOnFailure"); - digester.addObjectCreate("*/component/export", Export.class); - digester.addBeanPropertySetter("*/component/export/name"); - digester.addBeanPropertySetter("*/component/export/value"); - digester.addSetNext("*/component/export", "addExport"); + digester.addBeanPropertySetter("*/component/appExports"); + digester.addObjectCreate("*/component/componentExport", ComponentExport.class); + digester.addBeanPropertySetter("*/component/componentExport/name"); + digester.addBeanPropertySetter("*/component/componentExport/value"); + digester.addSetNext("*/component/componentExport", "addComponentExport"); digester.addSetNext("*/component", "addComponent"); digester.addObjectCreate("*/commandScript", CommandScript.class); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 20c16ff..1ea11a6 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -43,6 +43,7 @@ import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.apache.slider.providers.agent.application.metadata.Component; +import org.apache.slider.providers.agent.application.metadata.ComponentExport; import org.apache.slider.providers.agent.application.metadata.Export; import org.apache.slider.providers.agent.application.metadata.ExportGroup; import org.apache.slider.providers.agent.application.metadata.Metainfo; @@ -138,6 +139,7 @@ public class TestAgentProviderService { + " <name>HBASE_MASTER</name>\n" + " <category>MASTER</category>\n" + " <autoStartOnFailure>true</autoStartOnFailure>\n" + + " <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n" + " <minInstanceCount>1</minInstanceCount>\n" + " <maxInstanceCount>2</maxInstanceCount>\n" + " <commandScript>\n" @@ -155,16 +157,16 @@ public class TestAgentProviderService { + " <script>scripts/hbase_regionserver.py</script>\n" + " <scriptType>PYTHON</scriptType>\n" + " </commandScript>\n" - + " <exports>\n" - + " <export>\n" + + " <componentExports>\n" + + " <componentExport>\n" + " <name>PropertyA</name>\n" + " <value>${THIS_HOST}:${site.global.listen_port}</value>\n" - + " </export>\n" - + " <export>\n" + + " </componentExport>\n" + + " <componentExport>\n" + " <name>PropertyB</name>\n" + " <value>AConstant</value>\n" - + " </export>\n" - + " </exports>\n" + + " </componentExport>\n" + + " </componentExport>\n" + " </component>\n" + " </components>\n" + " <osSpecifics>\n" @@ -496,7 +498,8 @@ public class TestAgentProviderService { Assert.assertEquals(component.getMaxInstanceCount(), "2"); Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py"); Assert.assertEquals(component.getCategory(), "MASTER"); - Assert.assertEquals(component.getExports().size(), 0); + Assert.assertEquals(component.getComponentExports().size(), 0); + Assert.assertEquals(component.getAppExports(), "QuickLinks-JMX_Endpoint,QuickLinks-Master_Status"); found++; } if (component.getName().equals("HBASE_REGIONSERVER")) { @@ -506,9 +509,9 @@ public class TestAgentProviderService { Assert.assertNull(component.getMaxInstanceCount()); Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py"); Assert.assertEquals(component.getCategory(), "SLAVE"); - Assert.assertEquals(component.getExports().size(), 2); - List<Export> es = component.getExports(); - Export e = es.get(0); + Assert.assertEquals(component.getComponentExports().size(), 2); + List<ComponentExport> es = component.getComponentExports(); + ComponentExport e = es.get(0); Assert.assertEquals(e.getName(), "PropertyA"); Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}"); e = es.get(1); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml index 3d24f96..d9004ad 100644 --- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml +++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml @@ -22,11 +22,67 @@ <name>STORM</name> <comment>Apache Hadoop Stream processing framework</comment> <version>0.9.1.2.1</version> + <exportedConfigs>storm-site</exportedConfigs> + + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>app.jmx</name> + <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> + </export> + <export> + <name>app.monitor</name> + <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> + </export> + <export> + <name>app.metrics</name> + <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>ganglia.ui</name> + <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>nimbus.url</name> + <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + + <commandOrders> + <commandOrder> + <command>NIMBUS-START</command> + <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED + </requires> + </commandOrder> + <commandOrder> + <command>SUPERVISOR-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>DRPC_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_REST_API-START</command> + <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_UI_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + </commandOrders> + <components> <component> <name>NIMBUS</name> <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports> <commandScript> <script>scripts/nimbus.py</script> <scriptType>PYTHON</scriptType> @@ -37,6 +93,8 @@ <component> <name>STORM_REST_API</name> <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-app.jmx</appExports> <commandScript> <script>scripts/rest_api.py</script> <scriptType>PYTHON</scriptType> @@ -47,6 +105,13 @@ <component> <name>SUPERVISOR</name> <category>SLAVE</category> + <autoStartOnFailure>true</autoStartOnFailure> + <componentExports> + <componentExport> + <name>log_viewer_port</name> + <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value> + </componentExport> + </componentExports> <commandScript> <script>scripts/supervisor.py</script> <scriptType>PYTHON</scriptType> @@ -57,6 +122,9 @@ <component> <name>STORM_UI_SERVER</name> <category>MASTER</category> + <publishConfig>true</publishConfig> + <appExports>QuickLinks-app.monitor</appExports> + <autoStartOnFailure>true</autoStartOnFailure> <commandScript> <script>scripts/ui_server.py</script> <scriptType>PYTHON</scriptType> @@ -67,6 +135,7 @@ <component> <name>DRPC_SERVER</name> <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> <commandScript> <script>scripts/drpc_server.py</script> <scriptType>PYTHON</scriptType> @@ -86,10 +155,5 @@ </packages> </osSpecific> </osSpecifics> - - <configuration-dependencies> - <config-type>storm-site</config-type> - <config-type>global</config-type> - </configuration-dependencies> </application> </metainfo>
