SLIDER-96. Related fix. Dynamically create app packages for tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b64e1cdf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b64e1cdf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b64e1cdf Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it Commit: b64e1cdfc5d4d5c89c786a301209bd43391d5b36 Parents: 4eb8ea9 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Wed Jun 25 14:07:59 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Jun 26 13:41:31 2014 -0700 ---------------------------------------------------------------------- app-packages/accumulo/metainfo.xml | 238 ++++++++++--------- app-packages/hbase/metainfo.xml | 168 +++++++------ app-packages/storm-v0_91/metainfo.xml | 229 +++++++++--------- .../providers/agent/AgentClientProvider.java | 23 +- .../agent/AgentMiniClusterTestBase.groovy | 41 +++- .../slider/providers/agent/AgentTestBase.groovy | 44 +++- .../agent/TestAgentAMManagementWS.groovy | 3 +- .../slider/providers/agent/TestAgentEcho.groovy | 6 +- .../providers/agent/TestBuildBasicAgent.groovy | 17 +- .../curator/TestRegistryRestResources.groovy | 3 +- .../publisher/TestPublisherRestResources.groovy | 3 +- slider-core/src/test/python/appdef_1.zip | Bin 972 -> 0 bytes slider-core/src/test/python/metainfo.xml | 68 +++--- 13 files changed, 445 insertions(+), 398 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/accumulo/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml index eba75c6..4cf6c79 100644 --- a/app-packages/accumulo/metainfo.xml +++ b/app-packages/accumulo/metainfo.xml @@ -17,129 +17,131 @@ --> <metainfo> <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>ACCUMULO</name> - <comment> - The Apache Accumulo sorted, distributed key/value store is a robust, - scalable, high performance data storage system that features cell-based - access control and customizable server-side processing. It is based on - Google's BigTable design and is built on top of Apache Hadoop, - Zookeeper, and Thrift. - Requirements: - 1. Ensure parent dir for path (accumulo-site/instance.volumes) is accessible to the App owner. - </comment> - <version>${accumulo.version}</version> - <exportGroups> - <exportGroup> - <name>QuickLinks</name> - <exports> - <export> - <name>org.apache.slider.monitor</name> - <value>${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}</value> - </export> - <export> - <name>org.apache.slider.jmx</name> - <value>${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml</value> - </export> - </exports> - </exportGroup> - </exportGroups> - <commandOrders> - <commandOrder> - <command>ACCUMULO_MASTER-START</command> - <requires>ACCUMULO_MONITOR-INSTALLED</requires> - </commandOrder> - <commandOrder> - <command>ACCUMULO_TSERVER-START</command> - <requires>ACCUMULO_MASTER-STARTED</requires> - </commandOrder> - <commandOrder> - <command>ACCUMULO_MONITOR-START</command> - <requires>ACCUMULO_MASTER-STARTED</requires> - </commandOrder> - <commandOrder> - <command>ACCUMULO_GC-START</command> - <requires>ACCUMULO_MASTER-STARTED</requires> - </commandOrder> - <commandOrder> - <command>ACCUMULO_TRACER-START</command> - <requires>ACCUMULO_MASTER-STARTED</requires> - </commandOrder> - </commandOrders> - <components> - <component> - <name>ACCUMULO_MASTER</name> - <category>MASTER</category> - <commandScript> - <script>scripts/accumulo_master.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <application> + <name>ACCUMULO</name> + <comment> + The Apache Accumulo sorted, distributed key/value store is a robust, + scalable, high performance data storage system that features cell-based + access control and customizable server-side processing. It is based on + Google's BigTable design and is built on top of Apache Hadoop, + Zookeeper, and Thrift. + Requirements: + 1. Ensure parent dir for path (accumulo-site/instance.volumes) is accessible to the App owner. + </comment> + <version>${accumulo.version}</version> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.monitor</name> + <value> + ${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client} + </value> + </export> + <export> + <name>org.apache.slider.jmx</name> + <value> + ${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml + </value> + </export> + </exports> + </exportGroup> + </exportGroups> + <commandOrders> + <commandOrder> + <command>ACCUMULO_MASTER-START</command> + <requires>ACCUMULO_MONITOR-INSTALLED</requires> + </commandOrder> + <commandOrder> + <command>ACCUMULO_TSERVER-START</command> + <requires>ACCUMULO_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>ACCUMULO_MONITOR-START</command> + <requires>ACCUMULO_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>ACCUMULO_GC-START</command> + <requires>ACCUMULO_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>ACCUMULO_TRACER-START</command> + <requires>ACCUMULO_MASTER-STARTED</requires> + </commandOrder> + </commandOrders> + <components> + <component> + <name>ACCUMULO_MASTER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/accumulo_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>ACCUMULO_MONITOR</name> - <category>MASTER</category> - <publishConfig>true</publishConfig> - <commandScript> - <script>scripts/accumulo_monitor.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>ACCUMULO_MONITOR</name> + <category>MASTER</category> + <publishConfig>true</publishConfig> + <commandScript> + <script>scripts/accumulo_monitor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>ACCUMULO_GC</name> - <category>MASTER</category> - <commandScript> - <script>scripts/accumulo_gc.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>ACCUMULO_GC</name> + <category>MASTER</category> + <commandScript> + <script>scripts/accumulo_gc.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>ACCUMULO_TRACER</name> - <category>MASTER</category> - <commandScript> - <script>scripts/accumulo_tracer.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>ACCUMULO_TRACER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/accumulo_tracer.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>ACCUMULO_TSERVER</name> - <category>SLAVE</category> - <commandScript> - <script>scripts/accumulo_tserver.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - </component> + <component> + <name>ACCUMULO_TSERVER</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/accumulo_tserver.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> - <component> - <name>ACCUMULO_CLIENT</name> - <category>CLIENT</category> - <commandScript> - <script>scripts/accumulo_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - </component> - </components> + <component> + <name>ACCUMULO_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/accumulo_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + </components> - <osSpecifics> - <osSpecific> - <osType>any</osType> - <packages> - <package> - <type>tarball</type> - <name>files/accumulo-${accumulo.version}-bin.tar.gz</name> - </package> - </packages> - </osSpecific> - </osSpecifics> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/accumulo-${accumulo.version}-bin.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> - </service> - </services> + </application> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/hbase/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml index d5bb2d5..99413f6 100644 --- a/app-packages/hbase/metainfo.xml +++ b/app-packages/hbase/metainfo.xml @@ -17,93 +17,91 @@ --> <metainfo> <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>HBASE</name> - <comment> - Apache HBase is the Hadoop database, a distributed, scalable, big data store. - Requirements: - 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner. - 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance. - </comment> - <version>${hbase.version}</version> - <type>YARN-APP</type> - <minHadoopVersion>2.1.0</minHadoopVersion> - <exportGroups> - <exportGroup> - <name>QuickLinks</name> - <exports> - <export> - <name>org.apache.slider.jmx</name> - <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> - </export> - <export> - <name>org.apache.slider.monitor</name> - <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> - </export> - <export> - <name>org.apache.slider.metrics</name> - <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> - </export> - <export> - <name>org.apache.slider.ganglia</name> - <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> - </export> - </exports> - </exportGroup> - </exportGroups> - <commandOrders> - <commandOrder> - <command>HBASE_REGIONSERVER-START</command> - <requires>HBASE_MASTER-STARTED</requires> - </commandOrder> - </commandOrders> - <components> - <component> - <name>HBASE_MASTER</name> - <category>MASTER</category> - <minInstanceCount>1</minInstanceCount> - <maxInstanceCount>2</maxInstanceCount> - <commandScript> - <script>scripts/hbase_master.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <application> + <name>HBASE</name> + <comment> + Apache HBase is the Hadoop database, a distributed, scalable, big data store. + Requirements: + 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner. + 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance. + </comment> + <version>${hbase.version}</version> + <type>YARN-APP</type> + <minHadoopVersion>2.1.0</minHadoopVersion> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.jmx</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </export> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </export> + <export> + <name>org.apache.slider.metrics</name> + <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>org.apache.slider.ganglia</name> + <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + <commandOrders> + <commandOrder> + <command>HBASE_REGIONSERVER-START</command> + <requires>HBASE_MASTER-STARTED</requires> + </commandOrder> + </commandOrders> + <components> + <component> + <name>HBASE_MASTER</name> + <category>MASTER</category> + <minInstanceCount>1</minInstanceCount> + <maxInstanceCount>2</maxInstanceCount> + <commandScript> + <script>scripts/hbase_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>HBASE_REGIONSERVER</name> - <category>SLAVE</category> - <minInstanceCount>1</minInstanceCount> - <commandScript> - <script>scripts/hbase_regionserver.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - </component> + <component> + <name>HBASE_REGIONSERVER</name> + <category>SLAVE</category> + <minInstanceCount>1</minInstanceCount> + <commandScript> + <script>scripts/hbase_regionserver.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> - <component> - <name>HBASE_CLIENT</name> - <category>CLIENT</category> - <minInstanceCount>0</minInstanceCount> - <commandScript> - <script>scripts/hbase_client.py</script> - <scriptType>PYTHON</scriptType> - </commandScript> - </component> - </components> + <component> + <name>HBASE_CLIENT</name> + <category>CLIENT</category> + <minInstanceCount>0</minInstanceCount> + <commandScript> + <script>scripts/hbase_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + </components> - <osSpecifics> - <osSpecific> - <osType>any</osType> - <packages> - <package> - <type>tarball</type> - <name>files/hbase-${hbase.version}-bin.tar.gz</name> - </package> - </packages> - </osSpecific> - </osSpecifics> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/hbase-${hbase.version}-bin.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> - </service> - </services> + </application> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/storm-v0_91/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-v0_91/metainfo.xml b/app-packages/storm-v0_91/metainfo.xml index 2b10c52..7edd794 100644 --- a/app-packages/storm-v0_91/metainfo.xml +++ b/app-packages/storm-v0_91/metainfo.xml @@ -18,129 +18,128 @@ <metainfo> <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>STORM</name> - <comment>Apache Hadoop Stream processing framework</comment> - <version>0.9.1.2.1</version> + <application> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>0.9.1.2.1</version> - <exportGroups> - <exportGroup> - <name>QuickLinks</name> - <exports> - <export> - <name>org.apache.slider.jmx</name> - <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> - </export> - <export> - <name>org.apache.slider.monitor</name> - <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> - </export> - <export> - <name>org.apache.slider.metrics</name> - <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> - </export> - <export> - <name>org.apache.slider.ganglia</name> - <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> - </export> - </exports> - </exportGroup> - </exportGroups> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.jmx</name> + <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> + </export> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> + </export> + <export> + <name>org.apache.slider.metrics</name> + <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>org.apache.slider.ganglia</name> + <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + </export> + </exports> + </exportGroup> + </exportGroups> - <commandOrders> - <commandOrder> - <command>NIMBUS-START</command> - <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED</requires> - </commandOrder> - <commandOrder> - <command>SUPERVISOR-START</command> - <requires>NIMBUS-STARTED</requires> - </commandOrder> - <commandOrder> - <command>DRPC_SERVER-START</command> - <requires>NIMBUS-STARTED</requires> - </commandOrder> - <commandOrder> - <command>STORM_REST_API-START</command> - <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> - </commandOrder> - <commandOrder> - <command>STORM_UI_SERVER-START</command> - <requires>NIMBUS-STARTED</requires> - </commandOrder> - </commandOrders> + <commandOrders> + <commandOrder> + <command>NIMBUS-START</command> + <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED + </requires> + </commandOrder> + <commandOrder> + <command>SUPERVISOR-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>DRPC_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_REST_API-START</command> + <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_UI_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + </commandOrders> - <components> + <components> - <component> - <name>NIMBUS</name> - <category>MASTER</category> - <commandScript> - <script>scripts/nimbus.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>NIMBUS</name> + <category>MASTER</category> + <commandScript> + <script>scripts/nimbus.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>STORM_REST_API</name> - <category>MASTER</category> - <commandScript> - <script>scripts/rest_api.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>STORM_REST_API</name> + <category>MASTER</category> + <commandScript> + <script>scripts/rest_api.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>SUPERVISOR</name> - <category>SLAVE</category> - <commandScript> - <script>scripts/supervisor.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>SUPERVISOR</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/supervisor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>STORM_UI_SERVER</name> - <category>MASTER</category> - <publishConfig>true</publishConfig> - <commandScript> - <script>scripts/ui_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> + <component> + <name>STORM_UI_SERVER</name> + <category>MASTER</category> + <publishConfig>true</publishConfig> + <commandScript> + <script>scripts/ui_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> - <component> - <name>DRPC_SERVER</name> - <category>MASTER</category> - <commandScript> - <script>scripts/drpc_server.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - </components> + <component> + <name>DRPC_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/drpc_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> - <osSpecifics> - <osSpecific> - <osType>any</osType> - <packages> - <package> - <type>tarball</type> - <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> - </package> - </packages> - </osSpecific> - </osSpecifics> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> - <configuration-dependencies> - <config-type>storm-site</config-type> - <config-type>global</config-type> - </configuration-dependencies> - </service> - </services> + <configuration-dependencies> + <config-type>storm-site</config-type> + <config-type>global</config-type> + </configuration-dependencies> + </application> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index ea1c1b7..f3e2dd7 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -202,18 +202,25 @@ public class AgentClientProvider extends AbstractClientProvider public Set<String> getApplicationTags(SliderFileSystem fileSystem, String appDef) throws SliderException { Set<String> tags; + Metainfo metainfo; try { - Metainfo metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef); - Application application = metainfo.getApplication(); - tags = new HashSet<>(); - tags.add("Name: " + application.getName()); - tags.add("Version: " + application.getVersion()); - tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80)); + metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef); } catch (IOException e) { - log.error("error retrieving metainfo from {}", appDef, e); - throw new SliderException("error retrieving metainfo", e); + log.error("Error retrieving metainfo from {}", appDef, e); + throw new SliderException("Error retrieving metainfo", e); } + if(metainfo == null) { + log.error("Error retrieving metainfo from {}", appDef); + throw new SliderException("Error parsing metainfo file, possibly bad structure."); + } + + Application application = metainfo.getApplication(); + tags = new HashSet<>(); + tags.add("Name: " + application.getName()); + tags.add("Version: " + application.getVersion()); + tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80)); + return tags; } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy index cea3d74..8a4e5d8 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy @@ -20,13 +20,19 @@ package org.apache.slider.agent import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.apache.commons.compress.utils.IOUtils +import org.apache.commons.io.FileUtils import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderXMLConfKeysForTesting import org.apache.slider.common.params.Arguments import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.providers.agent.AgentKeys import org.apache.slider.test.YarnZKMiniClusterTestBase +import org.junit.AfterClass import org.junit.BeforeClass +import org.junit.rules.TemporaryFolder /** * test base for agent clusters @@ -34,28 +40,52 @@ import org.junit.BeforeClass @CompileStatic @Slf4j public abstract class AgentMiniClusterTestBase - extends YarnZKMiniClusterTestBase { +extends YarnZKMiniClusterTestBase { protected static File agentConf protected static File agentDef protected static File imagePath - protected static Map<String, String> agentDefOptions + protected static Map<String, String> agentDefOptions + private static TemporaryFolder tempFolder = new TemporaryFolder(); @BeforeClass public static void createSubConfFiles() { + File destDir = new File("target/agent_minicluster_testbase") destDir.mkdirs() agentConf = new File(destDir, "agentconf.zip") agentConf.createNewFile() agentDef = new File(destDir, "agentdef") agentDef.createNewFile() - File slider_dir = new File(new File(".").absoluteFile, "src/test/python"); - imagePath = new File(slider_dir, "appdef_1.zip") + + // dynamically create the app package for the test + tempFolder.create() + def pkgPath = tempFolder.newFolder("testpkg") + File imagePath = new File(pkgPath, "appdef_1.zip").canonicalFile + File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml"); + ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(imagePath)); + try { + zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name)); + IOUtils.copy(new FileInputStream(metainfo), zipFile); + zipFile.closeArchiveEntry(); + } + finally { + zipFile.close(); + } + agentDefOptions = [ - (AgentKeys.APP_DEF) : imagePath.toURI().toString(), + (AgentKeys.APP_DEF): imagePath.toURI().toString(), (AgentKeys.AGENT_CONF): agentConf.toURI().toString() ] } + @AfterClass + public static void cleanSubConfFiles() { + if (tempFolder.getRoot().exists()) { + FileUtils.deleteDirectory(tempFolder.getRoot()); + } + } + + @Override public String getTestConfigurationPath() { return "src/main/resources/" + AgentKeys.CONF_RESOURCE; @@ -74,7 +104,6 @@ public abstract class AgentMiniClusterTestBase void teardown() { super.teardown(); if (teardownKillall) { - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy index b6d0c6c..bb3eabe 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy @@ -20,11 +20,17 @@ package org.apache.slider.providers.agent import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.apache.commons.compress.utils.IOUtils import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.client.SliderClient import org.apache.slider.common.params.SliderActions import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.test.YarnZKMiniClusterTestBase +import org.junit.Before +import org.junit.Rule +import org.junit.rules.TemporaryFolder import static org.apache.slider.common.SliderXMLConfKeysForTesting.* import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE @@ -36,17 +42,31 @@ import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE @Slf4j public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { - public static - final int AGENT_CLUSTER_STARTUP_TIME = 1000 * DEFAULT_AGENT_LAUNCH_TIME_SECONDS - - /** - * The time to sleep before trying to talk to the HBase Master and - * expect meaningful results. - */ - public static - final int AGENT_CLUSTER_STARTUP_TO_LIVE_TIME = AGENT_CLUSTER_STARTUP_TIME - public static final int AGENT_GO_LIVE_TIME = 60000 - + public static final int AGENT_CLUSTER_STARTUP_TIME = 1000 * DEFAULT_AGENT_LAUNCH_TIME_SECONDS + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + public static String app_def_pkg_path; + + @Before + public void setupAppPkg() { + if (app_def_pkg_path == null) { + def pkgPath = folder.newFolder("testpkg") + File zipFileName = new File(pkgPath, "appdef_1.zip").canonicalFile + File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml"); + ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(zipFileName)); + try { + zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name)); + IOUtils.copy(new FileInputStream(metainfo), zipFile); + zipFile.closeArchiveEntry(); + } + finally { + zipFile.close(); + } + app_def_pkg_path = zipFileName.absolutePath + } + } @Override public String getTestConfigurationPath() { @@ -102,7 +122,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { boolean deleteExistingData, boolean create, boolean blockUntilRunning) { - + YarnConfiguration conf = testConfiguration http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy index de20e10..51d9bd6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy @@ -58,8 +58,7 @@ class TestAgentAMManagementWS extends AgentTestBase { false) Map<String, Integer> roles = [:] File slider_core = new File(new File(".").absoluteFile, "src/test/python"); - String app_def = "appdef_1.zip" - File app_def_path = new File(slider_core, app_def) + File app_def_path = new File(app_def_pkg_path) String agt_ver = "version" File agt_ver_path = new File(slider_core, agt_ver) String agt_conf = "agent.ini" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index 1072ebe..2f03b09 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -36,10 +36,9 @@ import static org.apache.slider.providers.agent.AgentKeys.* @Slf4j class TestAgentEcho extends AgentTestBase { - @Override void checkTestAssumptions(YarnConfiguration conf) { - + } @Test @@ -57,8 +56,7 @@ class TestAgentEcho extends AgentTestBase { File slider_core = new File(new File(".").absoluteFile, "src/test/python"); String echo_py = "echo.py" File echo_py_path = new File(slider_core, echo_py) - String app_def = "appdef_1.zip" - File app_def_path = new File(slider_core, app_def) + File app_def_path = new File(app_def_pkg_path) String agt_ver = "version" File agt_ver_path = new File(slider_core, agt_ver) String agt_conf = "agent.ini" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy index a597707..1879f84 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy @@ -46,30 +46,29 @@ class TestBuildBasicAgent extends AgentTestBase { private static class TestResources { static File slider_core = new File(new File(".").absoluteFile, "src/test/python"); - static String app_def = "appdef_1.zip" static String bad_app_def = "appdef_1.tar" - static File app_def_path = new File(slider_core, app_def) static File bad_app_def_path = new File(slider_core, bad_app_def) static String agt_conf = "agent.ini" static File agt_conf_path = new File(slider_core, agt_conf) - static public File getAppDef() { - return app_def_path; + public static File getAppDef() { + return new File(app_def_pkg_path); } - static public File getBadAppDef() { + public static File getBadAppDef() { return bad_app_def_path; } - static public File getAgentConf() { + public static File getAgentConf() { return agt_conf_path; } - static public File getAgentImg() { - return app_def_path; + public static File getAgentImg() { + return new File(app_def_pkg_path); } } + @Test public void testBuildMultipleRoles() throws Throwable { @@ -265,7 +264,7 @@ class TestBuildBasicAgent extends AgentTestBase { false) failWithBuildSucceeding(badArgs1, "missing package home or image path") } catch (BadConfigException expected) { - + } try { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy index 303cc73..4cc0f08 100644 --- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy @@ -75,8 +75,7 @@ class TestRegistryRestResources extends AgentTestBase { false) Map<String, Integer> roles = [:] File slider_core = new File(new File(".").absoluteFile, "src/test/python"); - String app_def = "appdef_1.zip" - File app_def_path = new File(slider_core, app_def) + File app_def_path = new File(app_def_pkg_path) String agt_ver = "version" File agt_ver_path = new File(slider_core, agt_ver) String agt_conf = "agent.ini" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy index 1309e57..cd7c9d8 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy @@ -58,8 +58,7 @@ class TestPublisherRestResources extends AgentTestBase { false) Map<String, Integer> roles = [:] File slider_core = new File(new File(".").absoluteFile, "src/test/python"); - String app_def = "appdef_1.zip" - File app_def_path = new File(slider_core, app_def) + File app_def_path = new File(app_def_pkg_path) String agt_ver = "version" File agt_ver_path = new File(slider_core, agt_ver) String agt_conf = "agent.ini" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/python/appdef_1.zip ---------------------------------------------------------------------- diff --git a/slider-core/src/test/python/appdef_1.zip b/slider-core/src/test/python/appdef_1.zip deleted file mode 100644 index 6ee6af4..0000000 Binary files a/slider-core/src/test/python/appdef_1.zip and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/python/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml index beefd9c..09b314e 100644 --- a/slider-core/src/test/python/metainfo.xml +++ b/slider-core/src/test/python/metainfo.xml @@ -17,41 +17,39 @@ --> <metainfo> <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>ECHO</name> - <comment> - Echo - </comment> - <version>0.1</version> - <type>YARN-APP</type> - <minHadoopVersion>2.1.0</minHadoopVersion> - <components> - <component> - <name>echo</name> - <category>MASTER</category> - <minInstanceCount>1</minInstanceCount> - <maxInstanceCount>2</maxInstanceCount> - <commandScript> - <script>echo.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - </components> - <osSpecifics> - <osSpecific> - <osType>any</osType> - <packages> - <package> - <type>tarball</type> - <name>files/echo.tar.gz</name> - </package> - </packages> - </osSpecific> - </osSpecifics> + <application> + <name>ECHO</name> + <comment> + Echo + </comment> + <version>0.1</version> + <type>YARN-APP</type> + <minHadoopVersion>2.1.0</minHadoopVersion> + <components> + <component> + <name>echo</name> + <category>MASTER</category> + <minInstanceCount>1</minInstanceCount> + <maxInstanceCount>2</maxInstanceCount> + <commandScript> + <script>echo.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/echo.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> - </service> - </services> + </application> </metainfo>