Author: stevel Date: Wed May 14 10:20:40 2014 New Revision: 1594532 URL: http://svn.apache.org/r1594532 Log: SLIDER-50 change URL for published key-val pairs to .filetype, not /filetype
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java Wed May 14 10:20:40 2014 @@ -18,10 +18,12 @@ package org.apache.slider.core.registry.docstore; +import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -35,7 +37,7 @@ import java.util.regex.Pattern; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) public class PublishedConfigSet { - public static final String VALID_NAME_PATTERN = "[A-Za-z0-9_-]+"; + public static final String VALID_NAME_PATTERN = RestPaths.PUBLISHED_CONFIGURATION_REGEXP; public static final String E_INVALID_NAME = "Invalid configuration name -it must match the pattern " + VALID_NAME_PATTERN; @@ -44,16 +46,25 @@ public class PublishedConfigSet { public Map<String, PublishedConfiguration> configurations = new HashMap<>(); + /** + * Put a name -it will be converted to lower case before insertion. + * Any existing entry will be overwritten (that includes an entry + * with a different case in the original name) + * @param name name of entry + * @param conf configuration + * @throws IllegalArgumentException if not a valid name + */ public void put(String name, PublishedConfiguration conf) { - validateName(name); - configurations.put(name, conf); + String name1 = name.toLowerCase(Locale.ENGLISH); + validateName(name1); + configurations.put(name1, conf); } /** * Validate the name -restricting it to the set defined in * {@link #VALID_NAME_PATTERN} * @param name name to validate - * @throws IllegalArgumentException if not + * @throws IllegalArgumentException if not a valid name */ public static void validateName(String name) { if (!validNames.matcher(name).matches()) { Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java Wed May 14 10:20:40 2014 @@ -158,4 +158,8 @@ public class PublishedConfiguration { sb.append('}'); return sb.toString(); } + + public PublishedConfigurationOutputter creatOutputter(ConfigFormat format) { + return PublishedConfigurationOutputter.createOutputter(format, this); + } } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java Wed May 14 10:20:40 2014 @@ -21,14 +21,20 @@ package org.apache.slider.core.registry. import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.slider.common.tools.ConfigHelper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; import java.util.Properties; +/** + * Output a published configuration + */ public abstract class PublishedConfigurationOutputter { protected final PublishedConfiguration owner; @@ -37,14 +43,41 @@ public abstract class PublishedConfigura this.owner = owner; } + /** + * Save the config to a destination file, in the format of this outputter + * @param dest destination file + * @throws IOException + */ public void save(File dest) throws IOException { - - } - - public String asString() throws IOException { - return ""; + try(FileOutputStream out = new FileOutputStream(dest)) { + save(out); + out.close(); + } } + /** + * Save the content. The default saves the asString() value + * to the output stream + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + /** + * Convert to a string + * @return + * @throws IOException + */ + public abstract String asString() throws IOException; + + /** + * Create an outputter for the chosen format + * @param format format enumeration + * @param owner owning config + * @return the outputter + */ + public static PublishedConfigurationOutputter createOutputter(ConfigFormat format, PublishedConfiguration owner) { Preconditions.checkNotNull(owner); @@ -71,13 +104,8 @@ public abstract class PublishedConfigura } @Override - public void save(File dest) throws IOException { - FileOutputStream out = new FileOutputStream(dest); - try { - configuration.writeXml(out); - } finally { - out.close(); - } + public void save(OutputStream out) throws IOException { + configuration.writeXml(out); } @Override @@ -100,19 +128,15 @@ public abstract class PublishedConfigura } @Override - public void save(File dest) throws IOException { - FileOutputStream out = new FileOutputStream(dest); - try { - properties.store(out, ""); - } finally { - out.close(); - } - + public void save(OutputStream out) throws IOException { + properties.store(out, ""); } - @Override + public String asString() throws IOException { - return properties.toString(); + StringWriter sw = new StringWriter(); + properties.store(sw, ""); + return sw.toString(); } } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java Wed May 14 10:20:40 2014 @@ -122,10 +122,10 @@ public class RegistryRetriever { /** * Get a complete configuration, with all values - * @param name - * @param external - * @return - * @throws IOException + * @param name name of the configuration + * @param external flag to indicate that it is an external configuration + * @return the retrieved config + * @throws IOException IO problems */ public PublishedConfiguration retrieveConfiguration(String name, boolean external) throws IOException { Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java Wed May 14 10:20:40 2014 @@ -18,11 +18,11 @@ package org.apache.slider.server.appmaster.web; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpPrincipal; import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -40,7 +40,8 @@ import java.util.HashSet; import java.util.Set; public class SliderAmIpFilter implements Filter { - private static final Log LOG = LogFactory.getLog(SliderAmIpFilter.class); + protected static final Logger log = + LoggerFactory.getLogger(SliderAmIpFilter.class); public static final String PROXY_HOST = "PROXY_HOST"; public static final String PROXY_URI_BASE = "PROXY_URI_BASE"; @@ -68,8 +69,8 @@ public class SliderAmIpFilter implements try { proxyAddresses = new HashSet<String>(); for(InetAddress add : InetAddress.getAllByName(proxyHost)) { - if (LOG.isDebugEnabled()) { - LOG.debug("proxy address is: " + add.getHostAddress()); + if (log.isDebugEnabled()) { + log.debug("proxy address is: " + add.getHostAddress()); } proxyAddresses.add(add.getHostAddress()); } @@ -96,13 +97,14 @@ public class SliderAmIpFilter implements HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletResponse httpResp = (HttpServletResponse)resp; - if (LOG.isDebugEnabled()) { - LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr()); + if (log.isDebugEnabled()) { + log.debug("Remote address for request is: " + httpReq.getRemoteAddr()); } - if(!httpReq.getRequestURI().startsWith(wsContextRoot) && + String requestURI = httpReq.getRequestURI(); + if(!requestURI.startsWith(wsContextRoot) && !getProxyAddresses().contains(httpReq.getRemoteAddr())) { - String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + - httpReq.getRequestURI()); + String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + + requestURI); httpResp.sendRedirect(redirectUrl); return; } @@ -117,15 +119,20 @@ public class SliderAmIpFilter implements } } } - if(user == null) { - LOG.warn("Could not find "+WebAppProxyServlet.PROXY_USER_COOKIE_NAME - +" cookie, so user will not be set"); - chain.doFilter(req, resp); - } else { - final AmIpPrincipal principal = new AmIpPrincipal(user); - ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, - principal); - chain.doFilter(requestWrapper, resp); + try { + if (user == null) { + log.warn("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME + + " cookie, so user will not be set"); + chain.doFilter(req, resp); + } else { + final AmIpPrincipal principal = new AmIpPrincipal(user); + ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, + principal); + chain.doFilter(requestWrapper, resp); + } + } catch (IOException | ServletException e) { + log.warn("When fetching {}: {}", requestURI, e); + throw e; } } } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java Wed May 14 10:20:40 2014 @@ -44,4 +44,13 @@ public class RestPaths { public static final String REGISTRY_SERVICE = "v1/service"; public static final String REGISTRY_ANYSERVICE = "v1/anyservice"; + + /** + * The regular expressions used to define valid configuration names/url path + * fragments: {@value} + */ + public static final String PUBLISHED_CONFIGURATION_REGEXP + ="[a-z0-9][a-z0-9_\\+-]*"; + + } Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java (original) +++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java Wed May 14 10:20:40 2014 @@ -19,8 +19,10 @@ package org.apache.slider.server.appmaster.web.rest.publisher; import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; import org.apache.slider.server.appmaster.web.WebAppApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +36,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriInfo; import java.io.IOException; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; /** * This publishes configuration sets @@ -42,7 +45,9 @@ public class PublisherResource { protected static final Logger log = LoggerFactory.getLogger(PublisherResource.class); private final WebAppApi slider; - + private static final String CONFIG = + "{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}"; + public PublisherResource(WebAppApi slider) { this.slider = slider; } @@ -90,36 +95,50 @@ public class PublisherResource { return publishedConfig; } - @GET - @Path("/{config}/json") + @Path("/{config}.json") @Produces({MediaType.APPLICATION_JSON}) public String getConfigurationContentJson( @PathParam("config") String config, @Context UriInfo uriInfo, @Context HttpServletResponse res) throws IOException { - - // delegate (including init) - PublishedConfiguration publishedConfig = - getConfigurationInstance(config, uriInfo, res); - return publishedConfig.asJson(); + return getStringRepresentation(config, uriInfo, res, + ConfigFormat.JSON); } - @GET - @Path("/{config}/xml") + @Path("/{config}.xml") @Produces({MediaType.APPLICATION_XML}) public String getConfigurationContentXML( @PathParam("config") String config, @Context UriInfo uriInfo, @Context HttpServletResponse res) throws IOException { - + return getStringRepresentation(config, uriInfo, res, + ConfigFormat.XML); + } + + @GET + @Path("/{config}.properties") + @Produces({MediaType.APPLICATION_XML}) + public String getConfigurationContentProperties( + @PathParam("config") String config, + @Context UriInfo uriInfo, + @Context HttpServletResponse res) throws IOException { + + return getStringRepresentation(config, uriInfo, res, + ConfigFormat.PROPERTIES); + } + + public String getStringRepresentation(String config, + UriInfo uriInfo, + HttpServletResponse res, ConfigFormat format) throws IOException { // delegate (including init) PublishedConfiguration publishedConfig = getConfigurationInstance(config, uriInfo, res); - return publishedConfig.asConfigurationXML(); + PublishedConfigurationOutputter outputter = + publishedConfig.creatOutputter(format); + return outputter.asString(); } - - + } Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy Wed May 14 10:20:40 2014 @@ -172,12 +172,23 @@ class TestStandaloneRegistryAM extends A //get the XML - def yarnSiteXML = appendToURL(yarnSitePublisher, "xml") + def yarnSiteXML = yarnSitePublisher + ".xml" String confXML = GET(yarnSiteXML) log.info("Conf XML at $yarnSiteXML = \n $confXML") + String properties = GET(yarnSitePublisher + ".properties") + Properties parsedProps = new Properties() + parsedProps.load(new StringReader(properties)) + assert parsedProps.size() > 0 + def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS) + def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME) + assert rmAddrFromDownloadedProperties + assert rmHostnameFromDownloadedProperties + + String json = GET(yarnSitePublisher + ".json") + describe("Registry List") @@ -202,10 +213,14 @@ class TestStandaloneRegistryAM extends A def yarnSite = retriever.retrieveConfiguration(ARTIFACT_NAME, true) assert !yarnSite.empty def siteXML = yarnSite.asConfiguration() - def rmAddr = siteXML.get(YarnConfiguration.RM_ADDRESS) - assert rmAddr - - + def rmHostnameViaClientSideXML = parsedProps.get( + YarnConfiguration.RM_HOSTNAME) + assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties + def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS) + + /* TODO SLIDER-52 PublishedConfiguration XML conf values are not resolved until client-side + assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties + */ describe "Internal configurations" assert !retriever.hasConfigurations(false) try { @@ -233,9 +248,6 @@ class TestStandaloneRegistryAM extends A assert 0 == client.actionRegistry(registryArgs) - - - // listconf --internal registryArgs.list = false; registryArgs.listConf = true Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy?rev=1594532&r1=1594531&r2=1594532&view=diff ============================================================================== --- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy (original) +++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/registry/TestConfigSetNaming.groovy Wed May 14 10:20:40 2014 @@ -29,6 +29,7 @@ class TestConfigSetNaming { def assertValid(String name) { PublishedConfigSet.validateName(name) } + def assertInvalid(String name) { try { PublishedConfigSet.validateName(name) @@ -44,8 +45,8 @@ class TestConfigSetNaming { } @Test - public void testUpperCase() throws Throwable { - assertValid("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + public void testUpperCaseInvalid() throws Throwable { + assertInvalid("ABCDEFGHIJKLMNOPQRSTUVWXYZ") } @Test @@ -55,7 +56,7 @@ class TestConfigSetNaming { @Test public void testChars() throws Throwable { - assertValid("-_") + assertValid("a-_+") } @Test @@ -70,7 +71,11 @@ class TestConfigSetNaming { "'", "\u0000", "\u0f00", - "" + "key.value", + "-", + "+", + "_", + "?", ].each {String s -> assertInvalid(s)} } }