SLIDER-365 slider "resolve" command to retrieve service record
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfaea962 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfaea962 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfaea962 Branch: refs/heads/develop Commit: bfaea9629ed7aa11e6c882e4a7533cc081cfc9dc Parents: fa53ef9 Author: Steve Loughran <ste...@apache.org> Authored: Thu Oct 9 11:29:10 2014 -0700 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Oct 9 11:32:26 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 57 +++++------- .../slider/common/params/ActionResolveArgs.java | 95 ++++++++++++++++++-- .../TestStandaloneYarnRegistryAM.groovy | 64 ++++++++----- .../apache/slider/test/SliderTestUtils.groovy | 21 ++++- 4 files changed, 171 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 7daa2b8..2944e2a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; @@ -134,13 +135,11 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; import java.net.URISyntaxException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -149,7 +148,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; @@ -394,14 +392,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe YarnAppListClient = new YarnAppListClient(yarnClient, getUsername(), getConfig()); // create the filesystem - sliderFileSystem = new SliderFileSystem(getConfig()); - - // and the registry -/* - YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); -*/ - + sliderFileSystem = new SliderFileSystem(getConfig()); } /** @@ -2211,40 +2202,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe args.validate(); RegistryOperations operations = getRegistryOperations(); String path = args.path; - Collection<ServiceRecord> serviceRecords; ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal(); - File outputPath = args.out; try { if (args.list) { + File destDir = args.destdir; + Map<String, ServiceRecord> recordMap = listServiceRecords(operations, path); - serviceRecords = recordMap.values(); - // list records out - StringBuilder builder = new StringBuilder(1024); - for (Entry<String, ServiceRecord> recordEntry : recordMap - .entrySet()) { + + for (Entry<String, ServiceRecord> recordEntry : recordMap.entrySet()) { + String name = recordEntry.getKey(); ServiceRecord instance = recordEntry.getValue(); - builder.append("\"").append(recordEntry.getKey()).append("\":\n"); - builder.append(serviceRecordMarshal.toJson(instance)); - builder.append("}\n"); - } - String records = builder.toString(); - if (outputPath == null) { - print(records); - } else { - SliderUtils.write(outputPath, records.getBytes("UTF-8"), false); + String json = serviceRecordMarshal.toJson(instance); + if (destDir == null) { + print(name); + print(json); + } else { + String filename = RegistryPathUtils.lastPathEntry(name) + ".json"; + File jsonFile = new File(destDir, filename); + SliderUtils.write(jsonFile, + serviceRecordMarshal.toBytes(instance), true); + } } } else { // resolve single entry ServiceRecord instance = resolve(path); - serviceRecords = new ArrayList<ServiceRecord>(1); - serviceRecords.add(instance); - // write out JSON content - if (outputPath != null) { - byte[] data = serviceRecordMarshal.toBytes(instance); - SliderUtils.write(outputPath, data, false); + File outFile = args.out; + if (args.destdir != null) { + outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path)); + } + if (outFile != null) { + SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true); } else { - // print to the console print(serviceRecordMarshal.toJson(instance)); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java index cf5611c..944be1b 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java @@ -20,7 +20,7 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import org.apache.slider.common.SliderKeys; +import org.apache.commons.lang.StringUtils; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.UsageException; @@ -45,10 +45,11 @@ public class ActionResolveArgs extends AbstractActionArgs { public static final String USAGE = "Usage: " + SliderActions.ACTION_RESOLVE + " " - + Arguments.ARG_PATH + " <path> " - + "[" + Arguments.ARG_LIST + "] " - + "[" + Arguments.ARG_VERBOSE + "] " - + "[" + Arguments.ARG_OUTPUT + " <filename> ] " + + ARG_PATH + " <path> " + + "[" + ARG_LIST + "] " + + "[" + ARG_VERBOSE + "] " + + "[" + ARG_OUTPUT + " <filename> ] " + + "[" + ARG_DESTDIR + " <directory> ] " ; public ActionResolveArgs() { } @@ -71,17 +72,95 @@ public class ActionResolveArgs extends AbstractActionArgs { description = "list services") public boolean list; - @Parameter(names = {ARG_PATH}, description = "resolve a path") public String path; + @Parameter(names = {ARG_DESTDIR}, + description = "destination directory for operations") + public File destdir; + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, - description = "Output destination") + description = "dest file") public File out; @Parameter(names = {ARG_VERBOSE}, description = "verbose output") public boolean verbose; - + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder(ACTION_RESOLVE).append(" "); + sb.append(ARG_PATH).append(" ").append(path).append(" "); + if (list) { + sb.append(ARG_LIST).append(" "); + } + if (destdir != null) { + sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" "); + } + if (out != null) { + sb.append(ARG_OUTPUT).append(" ").append(out).append(" "); + } + return sb.toString(); + } + + @Override + public void validate() throws BadCommandArgumentsException, UsageException { + super.validate(); + if (StringUtils.isEmpty(path)) { + throw new BadCommandArgumentsException("Missing mandatory argument " + + ARG_PATH); + } + if (list && out != null) { + throw new BadCommandArgumentsException("Argument " + + ARG_OUTPUT + + " not supported for " + ARG_LIST); + } + if (out != null && destdir != null) { + throw new BadCommandArgumentsException( + ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together" + ); + } + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public boolean isList() { + return list; + } + + public void setList(boolean list) { + this.list = list; + } + + public File getDestdir() { + return destdir; + } + + public void setDestdir(File destdir) { + this.destdir = destdir; + } + + public File getOut() { + return out; + } + + public void setOut(File out) { + this.out = out; + } + + public boolean isVerbose() { + return verbose; + } + + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index 84dcc50..ca200fd 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -20,6 +20,8 @@ package org.apache.slider.agent.standalone import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import org.apache.commons.io.IOUtils +import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.fs.PathNotFoundException import org.apache.hadoop.yarn.api.records.ApplicationReport import org.apache.hadoop.yarn.api.records.YarnApplicationState @@ -31,7 +33,10 @@ import org.apache.hadoop.registry.client.types.RegistryPathStatus import org.apache.hadoop.registry.client.types.ServiceRecord import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes import org.apache.slider.common.params.ActionResolveArgs +import org.apache.slider.common.params.Arguments +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.UnknownApplicationInstanceException +import org.apache.slider.core.main.LauncherExitCodes import static org.apache.hadoop.registry.client.binding.RegistryUtils.* import org.apache.slider.agent.AgentMiniClusterTestBase @@ -174,37 +179,52 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API) // use the resolve operation + describe "resolve CLI action" - ActionResolveArgs resolveArgs = new ActionResolveArgs() - resolveArgs.path = recordsPath; - resolveArgs.list = true; + File destDir= new File("target/resolve") + ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal() + FileUtil.fullyDelete(destDir) + File resolveListDir= new File(destDir, "list") + ActionResolveArgs resolveList = new ActionResolveArgs( + path:recordsPath, + list:true, + verbose:true) + // to stdout - client.actionResolve(resolveArgs) + client.actionResolve(resolveList) // to a file - File destFile = new File("target/resolve.json") - destFile.delete() - resolveArgs.out = destFile - client.actionResolve(resolveArgs) - assert destFile.exists() - destFile.delete() + resolveList.out = resolveListDir; + try { + client.actionResolve(resolveList) + } catch (BadCommandArgumentsException ex) { + assertExceptionDetails(ex, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, + Arguments.ARG_OUTPUT) + } + resolveList.out = null + resolveList.destdir = resolveListDir + client.actionResolve(resolveList) + File resolvedFile = new File(resolveListDir, clustername+".json") + assertFileExists("resolved file", resolvedFile) + def recordFromList = serviceRecordMarshal.fromFile(resolvedFile) // look at a single record - resolveArgs.out = null; - resolveArgs.list = false; - resolveArgs.path = recordsPath +"/"+ clustername + ActionResolveArgs resolveRecordCommand = new ActionResolveArgs( + path: recordsPath + "/" + clustername) // to stdout - client.actionResolve(resolveArgs) - resolveArgs.out = destFile - client.actionResolve(resolveArgs) - assert destFile.exists() - ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal() - def recordFromFile = serviceRecordMarshal.fromFile(destFile) + client.actionResolve(resolveRecordCommand) + + // then to a file + FileUtil.fullyDelete(destDir) + File singleFile = new File(destDir, "singlefile.json") + singleFile.delete() + resolveRecordCommand.out = singleFile + client.actionResolve(resolveRecordCommand) + assertFileExists("\"slider $resolveRecordCommand\"", singleFile) + def recordFromFile = serviceRecordMarshal.fromFile(singleFile) assert recordFromFile[YarnRegistryAttributes.YARN_ID] == serviceRecord[YarnRegistryAttributes.YARN_ID] assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] == serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] - - // hit the registry web page def registryEndpoint = serviceRecord.getExternalEndpoint( @@ -402,7 +422,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { // create a new registry args with the defaults back in registryArgs = new ActionRegistryArgs(clustername) registryArgs.getConf = yarn_site_config - registryArgs.dest = outputDir + registryArgs.out = outputDir describe registryArgs.toString() client.actionRegistry(registryArgs) assert new File(outputDir, yarn_site_config + ".xml").exists() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 0b45a5d..d09d280 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -79,13 +79,13 @@ class SliderTestUtils extends Assert { } public static void skip(String message) { - log.warn("Skipping test: " + message) + log.warn("Skipping test: {}", message) Assume.assumeTrue(message, false); } public static void assume(boolean condition, String message) { if (!condition) { - log.warn("Skipping test: " + message) + log.warn("Skipping test: {}", message) Assume.assumeTrue(message, false); } } @@ -793,4 +793,21 @@ class SliderTestUtils extends Assert { public String toURIArg(File file) { file.absoluteFile.toURI().toString() } + + /** + * Assert a file exists; fails with a listing of the parent dir + * @param text text for front of message + * @param file file to look for + * @throws FileNotFoundException + */ + public void assertFileExists(String text, File file) { + if (!file.exists()) { + def parent = file.parentFile + def files = parent.list() + StringBuilder builder = new StringBuilder() + builder.append("${parent.absolutePath}:\n") + files.each { String name-> builder.append(" $name\n")} + throw new FileNotFoundException("$text: file $file not found in $builder") + } + } }
