better parsing of comma-separated things, including from CLI Main; and use a builder for quoted string tokenizer
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/5d025ace Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/5d025ace Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/5d025ace Branch: refs/heads/0.4.0 Commit: 5d025ace2f7cde127f0ea6ce34661b3a7d9249d1 Parents: 1607a94 Author: Alex Heneveld <[email protected]> Authored: Thu Sep 27 11:44:58 2012 -0400 Committer: Alex Heneveld <[email protected]> Committed: Fri Sep 28 00:21:23 2012 -0400 ---------------------------------------------------------------------- .../location/basic/LocationRegistry.java | 19 ++- .../util/text/QuotedStringTokenizer.java | 46 +++++-- .../location/basic/LocationResolverTest.groovy | 19 ++- .../util/text/QuotedStringTokenizerTest.java | 125 +++++++++++-------- usage/cli/src/main/java/brooklyn/cli/Main.java | 19 ++- 5 files changed, 145 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/5d025ace/core/src/main/java/brooklyn/location/basic/LocationRegistry.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/location/basic/LocationRegistry.java b/core/src/main/java/brooklyn/location/basic/LocationRegistry.java index 4d3ad70..8fe9393 100644 --- a/core/src/main/java/brooklyn/location/basic/LocationRegistry.java +++ b/core/src/main/java/brooklyn/location/basic/LocationRegistry.java @@ -14,6 +14,9 @@ import brooklyn.config.BrooklynProperties; import brooklyn.location.Location; import brooklyn.location.LocationResolver; import brooklyn.util.MutableMap; +import brooklyn.util.text.QuotedStringTokenizer; +import brooklyn.util.text.WildcardGlobs; +import brooklyn.util.text.WildcardGlobs.PhraseTreatment; public class LocationRegistry { @@ -63,6 +66,8 @@ public class LocationRegistry { /** * Expects a collection of strings being the spec for locations, returns a list of locations. + * Also allows single elements which are comma-separated lists of locations. + * <p> * For legacy compatibility this also accepts nested lists, but that is deprecated * (and triggers a warning). */ @@ -70,9 +75,15 @@ public class LocationRegistry { List<Location> result = new ArrayList<Location>(); for (Object id : ids) { if (id instanceof String) { - result.add(resolve((String) id)); + // if it as comma-separated list -- TODO with no comma in the brackets + List<String> l = expandCommaSeparateLocationList((String)id); + if (l.size()>1) id = l; } else if (id instanceof Iterable) { log.warn("LocationRegistry got list of list of location strings, "+ids+"; flattening"); + } + if (id instanceof String) { + result.add(resolve((String) id)); + } else if (id instanceof Iterable) { result.addAll(getLocationsById((Iterable<?>) id)); } else if (id instanceof Location) { result.add((Location) id); @@ -84,6 +95,12 @@ public class LocationRegistry { return result; } + private List<String> expandCommaSeparateLocationList(String id) { + return WildcardGlobs.getGlobsAfterBraceExpansion("{"+id+"}", false, PhraseTreatment.INTERIOR_NOT_EXPANDABLE, PhraseTreatment.INTERIOR_NOT_EXPANDABLE); + // don't do this, it tries to expand commas inside parentheses which is not good! +// QuotedStringTokenizer.builder().addDelimiterChars(",").tokenizeAll((String)id); + } + public Map getProperties() { return properties; } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/5d025ace/core/src/main/java/brooklyn/util/text/QuotedStringTokenizer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/util/text/QuotedStringTokenizer.java b/core/src/main/java/brooklyn/util/text/QuotedStringTokenizer.java index 036aa31..52b1f45 100644 --- a/core/src/main/java/brooklyn/util/text/QuotedStringTokenizer.java +++ b/core/src/main/java/brooklyn/util/text/QuotedStringTokenizer.java @@ -48,6 +48,30 @@ public class QuotedStringTokenizer { this.includeDelimiters = includeDelimiters; updateNextToken(); } + + public static class Builder { + private String quoteChars = DEFAULT_QUOTE_CHARS; + private boolean includeQuotes=true; + private String delimiterChars=DEFAULT_DELIMITERS; + private boolean includeDelimiters=false; + + public QuotedStringTokenizer build(String stringToTokenize) { + return new QuotedStringTokenizer(stringToTokenize, quoteChars, includeQuotes, delimiterChars, includeDelimiters); + } + public List<String> tokenizeAll(String stringToTokenize) { + return new QuotedStringTokenizer(stringToTokenize, quoteChars, includeQuotes, delimiterChars, includeDelimiters).remainderAsList(); + } + + public Builder quoteChars(String quoteChars) { this.quoteChars = quoteChars; return this; } + public Builder addQuoteChars(String quoteChars) { this.quoteChars = this.quoteChars + quoteChars; return this; } + public Builder includeQuotes(boolean includeQuotes) { this.includeQuotes = includeQuotes; return this; } + public Builder delimiterChars(String delimiterChars) { this.delimiterChars = delimiterChars; return this; } + public Builder addDelimiterChars(String delimiterChars) { this.delimiterChars = this.delimiterChars + delimiterChars; return this; } + public Builder includeDelimiters(boolean includeDelimiters) { this.includeDelimiters = includeDelimiters; return this; } + } + public static Builder builder() { + return new Builder(); + } String peekedNextToken = null; @@ -105,23 +129,19 @@ public class QuotedStringTokenizer { //skip delimeters } while (!includeDelimiters && token.matches("["+delimiters+"]+")); - if (token.indexOf('"')<0 && token.indexOf('\'')<0) { - //no quote - peekedNextToken = token; - return; - } - StringBuffer nextToken = new StringBuffer(token); - - while (hasOpenQuote(nextToken.toString(), quoteChars) && delegate.hasMoreTokens()) { - //keep appending until the quote is ended or there are no more quotes - nextToken.append(delegate.nextToken()); - } - + pullUntilValid(nextToken); peekedNextToken = nextToken.toString(); } - public static boolean hasOpenQuote(String stringToCheck) { + private void pullUntilValid(StringBuffer nextToken) { + while (hasOpenQuote(nextToken.toString(), quoteChars) && delegate.hasMoreTokens()) { + //keep appending until the quote is ended or there are no more quotes + nextToken.append(delegate.nextToken()); + } + } + + public static boolean hasOpenQuote(String stringToCheck) { return hasOpenQuote(stringToCheck, DEFAULT_QUOTE_CHARS); } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/5d025ace/core/src/test/java/brooklyn/location/basic/LocationResolverTest.groovy ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/location/basic/LocationResolverTest.groovy b/core/src/test/java/brooklyn/location/basic/LocationResolverTest.groovy index eb33539..bcb053b 100644 --- a/core/src/test/java/brooklyn/location/basic/LocationResolverTest.groovy +++ b/core/src/test/java/brooklyn/location/basic/LocationResolverTest.groovy @@ -67,10 +67,21 @@ public class LocationResolverTest { public void testAcceptsList() { new LocationRegistry().getLocationsById(["localhost"]); } - + + @Test + public void testRegistryCommaResolution() { + List<Location> l; + l = new LocationRegistry().getLocationsById(["byon:(hosts=\"192.168.1.{1,2}\")"]); + Assert.assertEquals(1, l.size()); + l = new LocationRegistry().getLocationsById(["aws-ec2:us-west,byon:(hosts=\"192.168.1.{1,2}\"),aws-ec2:us-east"]); + Assert.assertEquals(3, l.size()); + l = new LocationRegistry().getLocationsById(["aws-ec2:us-west,byon:(hosts=\"192.168.1.{1,2}\",user=bob),aws-ec2:us-east"]); + Assert.assertEquals(3, l.size()); + } + @Test public void testAcceptsListOLists() { - //accidental, but if inner list has a single item it automatically gets coerced correctly to string + //if inner list has a single item it automatically gets coerced correctly to string //preserve for compatibility with older CommandLineLocations (since 0.4.0) [but log warning] new LocationRegistry().getLocationsById([["localhost"]]); } @@ -82,9 +93,9 @@ public class LocationResolverTest { @Test public void testLegacyCommandLineAcceptsListOLists() { - //accidental, but if inner list has a single item it automatically gets coerced correctly to string + //if inner list has a single item it automatically gets coerced correctly to string //preserve for compatibility (since 0.4.0) CommandLineLocations.getLocationsById([["localhost"]]); } - + } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/5d025ace/core/src/test/java/brooklyn/util/text/QuotedStringTokenizerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/util/text/QuotedStringTokenizerTest.java b/core/src/test/java/brooklyn/util/text/QuotedStringTokenizerTest.java index e3ba325..bccfe2d 100644 --- a/core/src/test/java/brooklyn/util/text/QuotedStringTokenizerTest.java +++ b/core/src/test/java/brooklyn/util/text/QuotedStringTokenizerTest.java @@ -11,67 +11,86 @@ import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import junit.framework.Assert; + import org.testng.annotations.Test; -/** - * The ConfigParserTest - * - * @author aled - **/ public class QuotedStringTokenizerTest { - - // have to initialise to use the methods (instance as it can take custom tokens) - private QuotedStringTokenizer defaultTokenizer= new QuotedStringTokenizer("", true); - - @Test - public void testQuoting() throws Exception { - assertQuoteUnquoteFor("a=b"); - assertQuoteUnquoteFor("a=\"things\",b=c"); - assertQuoteUnquoteFor("thing=\"\""); - assertQuoteUnquoteFor("\"thing\"=\"\""); - assertQuoteUnquoteFor(""); + + // have to initialise to use the methods (instance as it can take custom tokens) + private QuotedStringTokenizer defaultTokenizer= new QuotedStringTokenizer("", true); + + @Test + public void testQuoting() throws Exception { + assertQuoteUnquoteFor("a=b"); + assertQuoteUnquoteFor("a=\"things\",b=c"); + assertQuoteUnquoteFor("thing=\"\""); + assertQuoteUnquoteFor("\"thing\"=\"\""); + assertQuoteUnquoteFor(""); assertQuoteUnquoteFor("\""); assertQuoteUnquoteFor("\"\""); - + assertUnquoteFor("", "''"); assertUnquoteFor("thing=", "\"thing\"=\"\""); assertUnquoteFor("a=", "a=\"\""); - } - - @Test - public void testTokenizing() throws Exception { - testResultingTokens("foo,bar,baz", "\"", false, ",", false, "foo", "bar", "baz"); - testResultingTokens("\"foo,bar\",baz", "\"", false, ",", false, "foo,bar", "baz"); - testResultingTokens("\"foo,,bar\",baz", "\"", false, ",", false, "foo,,bar", "baz"); - - // Have seen "the operator ""foo"" is not recognised" entries in BAML CSV files. - testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ",", false, "foo bar baz"); - testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ",", false, "foo bar baz"); - - // FIXME: would like to return empty tokens when we encounter adjacent delimiters, but need - // to work around brain-dead java.util.StringTokenizer to do this. - // testResultingTokens("foo,,baz", "\"", false, ",", false, "foo", "", "baz"); - } - - private void testResultingTokens(String input, String quoteChars, boolean includeQuotes, String delimiterChars, boolean includeDelimiters, String... expectedTokens) { - QuotedStringTokenizer tok = new QuotedStringTokenizer(input, quoteChars, includeQuotes, delimiterChars, includeDelimiters); - testResultingTokens(input, tok, expectedTokens); - } - - private void testResultingTokens(String input, QuotedStringTokenizer tok, String... expectedTokens) { - List<String> actual = new LinkedList<String>(); - while (tok.hasMoreTokens()) actual.add(tok.nextToken()); - assertEquals(actual, Arrays.asList(expectedTokens), "Wrong tokens returned."); - } - - private void assertQuoteUnquoteFor(String unquoted) { - String quoted = defaultTokenizer.quoteToken(unquoted); - String reunquoted = defaultTokenizer.unquoteToken(quoted); - //System.out.println("orig="+unquoted+" quoted="+quoted+" reunquoted="+reunquoted); - assertEquals(reunquoted, unquoted); - } - - private void assertUnquoteFor(String expected, String quoted) { + } + + @Test + public void testTokenizing() throws Exception { + testResultingTokens("foo,bar,baz", "\"", false, ",", false, "foo", "bar", "baz"); + testResultingTokens("\"foo,bar\",baz", "\"", false, ",", false, "foo,bar", "baz"); + testResultingTokens("\"foo,,bar\",baz", "\"", false, ",", false, "foo,,bar", "baz"); + + // Have seen "the operator ""foo"" is not recognised" entries in BAML CSV files. + testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ",", false, "foo bar baz"); + testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ",", false, "foo bar baz"); + + // FIXME: would like to return empty tokens when we encounter adjacent delimiters, but need + // to work around brain-dead java.util.StringTokenizer to do this. + // testResultingTokens("foo,,baz", "\"", false, ",", false, "foo", "", "baz"); + } + + @Test + public void testTokenizingBuilder() throws Exception { + Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().tokenizeAll("foo bar")); + Assert.assertEquals(Arrays.asList("foo,bar"), QuotedStringTokenizer.builder().tokenizeAll("foo,bar")); + Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().delimiterChars(",").tokenizeAll("foo,bar")); + Assert.assertEquals(Arrays.asList("foo", " bar"), QuotedStringTokenizer.builder().delimiterChars(",").tokenizeAll("foo, bar")); + Assert.assertEquals(Arrays.asList("foo", "bar"), QuotedStringTokenizer.builder().addDelimiterChars(",").tokenizeAll("foo, bar")); + } + + @Test + public void testCommaInQuotes() throws Exception { + List<String> l = QuotedStringTokenizer.builder().addDelimiterChars(",").tokenizeAll("location1,byon:(hosts=\"loc2,loc3\"),location4"); + Assert.assertEquals(Arrays.asList("location1", "byon:(hosts=\"loc2,loc3\")", "location4"), l); + } + + /** not implemented yet */ + @Test(enabled=false) + public void testCommaInParentheses() throws Exception { + List<String> l = QuotedStringTokenizer.builder().addDelimiterChars(",").tokenizeAll("location1, byon:(hosts=\"loc2,loc3\",user=foo),location4"); + Assert.assertEquals(Arrays.asList("location1", "byon:(hosts=\"loc2,loc3\",user=foo)", "location4"), l); + } + + private void testResultingTokens(String input, String quoteChars, boolean includeQuotes, String delimiterChars, boolean includeDelimiters, String... expectedTokens) { + QuotedStringTokenizer tok = new QuotedStringTokenizer(input, quoteChars, includeQuotes, delimiterChars, includeDelimiters); + testResultingTokens(input, tok, expectedTokens); + } + + private void testResultingTokens(String input, QuotedStringTokenizer tok, String... expectedTokens) { + List<String> actual = new LinkedList<String>(); + while (tok.hasMoreTokens()) actual.add(tok.nextToken()); + assertEquals(actual, Arrays.asList(expectedTokens), "Wrong tokens returned."); + } + + private void assertQuoteUnquoteFor(String unquoted) { + String quoted = defaultTokenizer.quoteToken(unquoted); + String reunquoted = defaultTokenizer.unquoteToken(quoted); + //System.out.println("orig="+unquoted+" quoted="+quoted+" reunquoted="+reunquoted); + assertEquals(reunquoted, unquoted); + } + + private void assertUnquoteFor(String expected, String quoted) { String unquoted = defaultTokenizer.unquoteToken(quoted); //System.out.println("expected="+expected+" quoted="+quoted+" unquoted="+unquoted); assertEquals(unquoted, expected); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/5d025ace/usage/cli/src/main/java/brooklyn/cli/Main.java ---------------------------------------------------------------------- diff --git a/usage/cli/src/main/java/brooklyn/cli/Main.java b/usage/cli/src/main/java/brooklyn/cli/Main.java index 21b6673..1b8c3db 100644 --- a/usage/cli/src/main/java/brooklyn/cli/Main.java +++ b/usage/cli/src/main/java/brooklyn/cli/Main.java @@ -37,7 +37,6 @@ import com.google.common.base.Objects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; public class Main { @@ -153,10 +152,6 @@ public class Main { if (verbose) { System.out.println("Launching brooklyn app: "+app+" in "+locations); } - if (locations==null) { - log.warn("Locations parameter not supplied. Assuming empty list."); - locations = ""; - } BrooklynLauncher launcher = BrooklynLauncher.newLauncher(); ResourceUtils utils = new ResourceUtils(this); @@ -174,6 +169,13 @@ public class Main { launcher.webconsolePort(port); launcher.webconsole(!noConsole); + if (locations==null || locations.isEmpty()) { + log.warn("Locations parameter not supplied: assuming localhost"); + locations = "localhost"; + } + // lean on getLocationsById to do parsing + List<Location> brooklynLocations = new LocationRegistry().getLocationsById(Arrays.asList(locations)); + // Create the instance of the brooklyn app AbstractApplication application = null; if (app!=null) { @@ -182,13 +184,6 @@ public class Main { launcher.managing(application); } - // Figure out the brooklyn location(s) where to launch the application - Iterable<String> parsedLocations = new QuotedStringTokenizer(locations).remainderAsList(); - log.info("Parsed user provided location(s): {}",Lists.newArrayList(parsedLocations)); - List<Location> brooklynLocations = new LocationRegistry().getLocationsById( - (parsedLocations==null || Iterables.isEmpty(parsedLocations)) ? - ImmutableSet.of(CommandLineLocations.LOCALHOST) : parsedLocations); - // Launch server log.info("Launching Brooklyn web console management"); launcher.launch();
