Author: jmaron Date: Fri May 16 20:10:20 2014 New Revision: 1595332 URL: http://svn.apache.org/r1595332 Log: SLIDER-59 added name, version, and description fields/tags to slider app listing and made minor unit test corrections for failing tests
Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java Fri May 16 20:10:20 2014 @@ -121,6 +121,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; /** * Client service for Slider @@ -688,13 +689,18 @@ public class SliderClient extends Abstra } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); + + // add the tags if available + Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, + appOperations.getGlobalOptions().get(AgentKeys.APP_DEF)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, sliderFileSystem, yarnClient, clusterSecure, - sliderAMResourceComponent); + sliderAMResourceComponent, + applicationTags); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java Fri May 16 20:10:20 2014 @@ -79,6 +79,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.TreeSet; @@ -525,9 +526,14 @@ public final class SliderUtils { public static String appReportToString(ApplicationReport r, String separator) { StringBuilder builder = new StringBuilder(512); builder.append("application ").append( - r.getName()).append("/").append(r.getApplicationType()); - builder.append(separator).append( - "state: ").append(r.getYarnApplicationState()); + r.getName()).append("/").append(r.getApplicationType()).append(separator); + Set<String> tags = r.getApplicationTags(); + if (!tags.isEmpty()) { + for (String tag : tags) { + builder.append(tag).append(separator); + } + } + builder.append("state: ").append(r.getYarnApplicationState()); builder.append(separator).append("URL: ").append(r.getTrackingUrl()); builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString()); long finishTime = r.getFinishTime(); Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java Fri May 16 20:10:20 2014 @@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; +import java.util.Set; public class AppMasterLauncher extends AbstractLauncher { @@ -73,7 +74,8 @@ public class AppMasterLauncher extends A CoreFileSystem fs, SliderYarnClientImpl yarnClient, boolean secureCluster, - Map<String, String> options + Map<String, String> options, + Set<String> applicationTags ) throws IOException, YarnException { super(conf, fs); this.yarnClient = yarnClient; @@ -88,6 +90,9 @@ public class AppMasterLauncher extends A submissionContext.setApplicationName(name); // app type used in service enum; submissionContext.setApplicationType(type); + if (!applicationTags.isEmpty()) { + submissionContext.setApplicationTags(applicationTags); + } extractResourceRequirements(resource, options); } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java Fri May 16 20:10:20 2014 @@ -32,7 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES; @@ -207,4 +209,13 @@ public abstract class AbstractClientProv validateInstanceDefinition(instanceDefinition); } + /** + * Return a set of application specific string tags. + * @return the set of tags. + */ + public Set<String> getApplicationTags (SliderFileSystem fileSystem, + String appDef) throws SliderException { + return Collections.emptySet(); + } + } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java Fri May 16 20:10:20 2014 @@ -34,11 +34,14 @@ import org.apache.slider.core.launch.Abs import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.providers.agent.application.metadata.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -194,4 +197,23 @@ public class AgentClientProvider extends IOException, SliderException { } + + @Override + public Set<String> getApplicationTags(SliderFileSystem fileSystem, + String appDef) throws SliderException { + Set<String> tags; + try { + Metainfo metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef); + Service service = metainfo.getServices().get(0); + tags = new HashSet<>(); + tags.add("Name: " + service.getName()); + tags.add("Version: " + service.getVersion()); + tags.add("Description: " + service.getComment()); + } catch (IOException e) { + log.error("error retrieving metainfo from {}", appDef, e); + throw new SliderException("error retrieving metainfo", e); + } + + return tags; + } } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java Fri May 16 20:10:20 2014 @@ -242,17 +242,7 @@ public class AgentProviderService extend protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, String appDef) throws IOException { - log.info("Reading metainfo at {}", appDef); - InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream( - fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml"); - if (metainfoStream == null) { - log.error("metainfo.xml is unavailable at {}.", appDef); - throw new IOException("metainfo.xml is required in app package."); - } - - Metainfo metainfo = new MetainfoParser().parse(metainfoStream); - - return metainfo; + return AgentUtils.getApplicationMetainfo(fileSystem, appDef); } protected void publishComponentConfiguration(String name, String description, Added: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java?rev=1595332&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java (added) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java Fri May 16 20:10:20 2014 @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.providers.agent.application.metadata.MetainfoParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; + +/** + * + */ +public class AgentUtils { + private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); + + static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, + String appDef) throws IOException { + log.info("Reading metainfo at {}", appDef); + InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream( + fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml"); + if (metainfoStream == null) { + log.error("metainfo.xml is unavailable at {}.", appDef); + throw new IOException("metainfo.xml is required in app package."); + } + + Metainfo metainfo = new MetainfoParser().parse(metainfoStream); + + return metainfo; + } + +} Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy Fri May 16 20:10:20 2014 @@ -194,7 +194,7 @@ class TestStandaloneRegistryAM extends A describe("Registry List") - log.info(GET(registryURL, RestPaths.REGISTRY_SERVICE )) + log.info(GET(registryURL)) describe "Registry Retrieval Class" Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy?rev=1595332&r1=1595331&r2=1595332&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy Fri May 16 20:10:20 2014 @@ -116,18 +116,17 @@ class TestRegistryRestResources extends //WS get Client client = createTestClient(); - // test the exposed WADL link WebResource webResource = client.resource(registry_url); - ClientResponse response = webResource.type(MediaType.APPLICATION_XML) - .get(ClientResponse.class); - assert response.status == 200 - assert response.type == (new MediaType("application", WADL)) + List<String> serviceList = webResource.type(MediaType.APPLICATION_JSON) + .get(List.class); + log.info("service list: {}", serviceList) + assert serviceList.contains(SliderKeys.APP_TYPE) // test the available GET URIs webResource = client.resource( appendToURL(registry_url, RestPaths.REGISTRY_SERVICE)); - - response = webResource.type(MediaType.APPLICATION_JSON) + + ClientResponse response = webResource.type(MediaType.APPLICATION_JSON) .get(ClientResponse.class); def responseStr = response.getEntity(String.class) log.info("response is " + responseStr) Added: incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java?rev=1595332&view=auto ============================================================================== --- incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java (added) +++ incubator/slider/trunk/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java Fri May 16 20:10:20 2014 @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.junit.Test; + +import java.io.InputStream; +import java.util.Set; + +/** + * + */ +public class TestAgentClientProvider { + @Test + public void testGetApplicationTags () throws Exception { + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + AgentClientProvider provider = new AgentClientProvider(null); + Set<String> tags = provider.getApplicationTags(sliderFileSystem, + "target/test-classes/org/apache/slider/common/tools/test.zip"); + assert tags != null; + assert !tags.isEmpty(); + assert tags.contains("Name: STORM"); + assert tags.contains("Description: Apache Hadoop Stream processing framework"); + assert tags.contains("Version: 0.9.1.2.1"); + + } +}