Repository: brooklyn-server Updated Branches: refs/heads/master b87c20b18 -> 404294956
State transformations: support deletions Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/49a4774a Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/49a4774a Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/49a4774a Branch: refs/heads/master Commit: 49a4774aad841acddf2889b335576fa65ad11d67 Parents: b87c20b Author: Aled Sage <aled.s...@gmail.com> Authored: Fri Jul 22 17:48:13 2016 +0100 Committer: Aled Sage <aled.s...@gmail.com> Committed: Fri Jul 22 17:48:13 2016 +0100 ---------------------------------------------------------------------- .../rebind/transformer/CompoundTransformer.java | 87 +++++++++++++++++++- .../transformer/CompoundTransformerLoader.java | 21 ++++- .../CompoundTransformerLoaderTest.java | 59 ++++++++++++- .../transformer/CompoundTransformerTest.java | 39 ++++++++- 4 files changed, 198 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/49a4774a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformer.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformer.java index 09b28b4..ef8531c 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformer.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformer.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.brooklyn.api.mgmt.rebind.RebindExceptionHandler; import org.apache.brooklyn.api.mgmt.rebind.mementos.BrooklynMementoRawData; @@ -32,17 +33,24 @@ import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.core.ResourceUtils; import org.apache.brooklyn.util.core.text.TemplateProcessor; import org.apache.brooklyn.util.text.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.Beta; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; @Beta public class CompoundTransformer { - + + private static final Logger LOG = LoggerFactory.getLogger(CompoundTransformer.class); + public static final CompoundTransformer NOOP = builder().build(); // TODO Does not yet handle BrooklynMementoTransformer, for changing an entire BrooklynMemento. @@ -55,6 +63,7 @@ public class CompoundTransformer { public static class Builder { private final Multimap<BrooklynObjectType, RawDataTransformer> rawDataTransformers = ArrayListMultimap.<BrooklynObjectType, RawDataTransformer>create(); + private final Multimap<BrooklynObjectType, String> deletions = HashMultimap.<BrooklynObjectType, String>create(); public Builder rawDataTransformer(RawDataTransformer val) { for (BrooklynObjectType type : BrooklynObjectType.values()) { @@ -204,15 +213,25 @@ public class CompoundTransformer { // xstream format for inner classes is like <org.apache.brooklyn.core.mgmt.rebind.transformer.CompoundTransformerTest_-OrigType> return (val.contains("$")) ? val.replace("$", "_-") : val; } + + /** Ids of items to be deleted from the persisted state */ + public Builder deletion(BrooklynObjectType type, Iterable<? extends String> ids) { + deletions.putAll(type, ids); + return this; + } + public CompoundTransformer build() { return new CompoundTransformer(this); } } private final Multimap<BrooklynObjectType, RawDataTransformer> rawDataTransformers; - + + private final Multimap<BrooklynObjectType, String> deletions; + protected CompoundTransformer(Builder builder) { rawDataTransformers = builder.rawDataTransformers; + deletions = builder.deletions; } public BrooklynMementoRawData transform(BrooklynMementoPersisterToObjectStore reader, RebindExceptionHandler exceptionHandler) throws Exception { @@ -231,6 +250,65 @@ public class CompoundTransformer { // TODO @neykov asks whether transformers should be run in registration order, // rather than in type order. TBD. (would be an easy change.) // (they're all strings so it shouldn't matter!) + + for (BrooklynObjectType type : BrooklynObjectType.values()) { + Set<String> itemsToDelete = ImmutableSet.copyOf(deletions.get(type)); + Set<String> missing; + switch (type) { + case ENTITY: + missing = Sets.difference(itemsToDelete, entities.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + entities.keySet().removeAll(itemsToDelete); + break; + case LOCATION: + missing = Sets.difference(itemsToDelete, locations.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + locations.keySet().removeAll(itemsToDelete); + break; + case POLICY: + missing = Sets.difference(itemsToDelete, policies.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + policies.keySet().removeAll(itemsToDelete); + break; + case ENRICHER: + missing = Sets.difference(itemsToDelete, enrichers.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + enrichers.keySet().removeAll(itemsToDelete); + break; + case FEED: + missing = Sets.difference(itemsToDelete, feeds.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + feeds.keySet().removeAll(itemsToDelete); + break; + case CATALOG_ITEM: + missing = Sets.difference(itemsToDelete, catalogItems.keySet()); + if (missing.size() > 0) { + LOG.warn("Unable to delete " + type + " id"+Strings.s(missing.size())+" ("+missing+"), " + + "because not found in persisted state (continuing)"); + } + catalogItems.keySet().removeAll(itemsToDelete); + break; + case UNKNOWN: + break; // no-op + default: + throw new IllegalStateException("Unexpected brooklyn object type "+type); + } + } for (BrooklynObjectType type : BrooklynObjectType.values()) { Collection<RawDataTransformer> transformers = rawDataTransformers.get(type); @@ -288,4 +366,9 @@ public class CompoundTransformer { Multimap<BrooklynObjectType, RawDataTransformer> getRawDataTransformers() { return ArrayListMultimap.create(rawDataTransformers); } + + @VisibleForTesting + Multimap<BrooklynObjectType, String> getDeletions() { + return HashMultimap.create(deletions); + } } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/49a4774a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoader.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoader.java index 37f2cae..2e4049d 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoader.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoader.java @@ -19,9 +19,12 @@ package org.apache.brooklyn.core.mgmt.rebind.transformer; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; +import org.apache.brooklyn.api.objs.BrooklynObjectType; import org.apache.brooklyn.core.mgmt.rebind.transformer.CompoundTransformer.Builder; import org.apache.brooklyn.util.core.ClassLoaderUtils; import org.apache.brooklyn.util.core.ResourceUtils; @@ -34,6 +37,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; @Beta public class CompoundTransformerLoader { @@ -56,8 +60,23 @@ public class CompoundTransformerLoader { return builder.build(); } + @SuppressWarnings("unchecked") private static void addRule(Builder builder, String name, Map<?,?> args) { - if (name.equals("renameClass")) { + if (name.equals("deletions")) { + Set<String> validKeys = Sets.newLinkedHashSet(); + for (BrooklynObjectType type : BrooklynObjectType.values()) { + String key = type.getSubPathName(); + validKeys.add(key); + List<String> ids = (List<String>) args.get(key); + if (ids != null) { + builder.deletion(type, ids); + } + } + Set<?> otherKeys = Sets.difference(args.keySet(), validKeys); + if (otherKeys.size() > 0) { + throw new IllegalStateException("Unsupported transform "+otherKeys+" in '"+name+"' ("+args+")"); + } + } else if (name.equals("renameClass")) { String oldVal = (String) args.get("old_val"); String newVal = (String) args.get("new_val"); builder.renameClass(oldVal, newVal); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/49a4774a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoaderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoaderTest.java b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoaderTest.java index a54d4ba..ecba1e8 100644 --- a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoaderTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerLoaderTest.java @@ -18,18 +18,20 @@ */ package org.apache.brooklyn.core.mgmt.rebind.transformer; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.util.Collection; import org.apache.brooklyn.api.objs.BrooklynObjectType; -import org.apache.brooklyn.core.mgmt.rebind.transformer.CompoundTransformer; -import org.apache.brooklyn.core.mgmt.rebind.transformer.CompoundTransformerLoader; -import org.apache.brooklyn.core.mgmt.rebind.transformer.RawDataTransformer; import org.apache.brooklyn.core.mgmt.rebind.transformer.impl.XsltTransformer; +import org.apache.brooklyn.test.Asserts; import org.testng.annotations.Test; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; public class CompoundTransformerLoaderTest { @@ -70,6 +72,57 @@ public class CompoundTransformerLoaderTest { assertTrue(Iterables.get(rawDataTransformers, 5) instanceof MyRawDataTransformer); } + @Test + public void testLoadsDeletionsFromYaml() throws Exception { + String contents = + "- deletions:\n"+ + " catalog:\n"+ + " - cat1\n"+ + " - cat2\n"+ + " entities:\n"+ + " - ent1\n"+ + " - ent2\n"+ + " locations:\n"+ + " - loc1\n"+ + " - loc2\n"+ + " enrichers:\n"+ + " - enricher1\n"+ + " - enricher2\n"+ + " policies:\n"+ + " - pol1\n"+ + " - pol2\n"+ + " feeds:\n"+ + " - feed1\n"+ + " - feed2\n"; + + CompoundTransformer transformer = CompoundTransformerLoader.load(contents); + + Multimap<BrooklynObjectType, String> deletions = transformer.getDeletions(); + HashMultimap<BrooklynObjectType, String> expected = HashMultimap.create(); + expected.putAll(BrooklynObjectType.CATALOG_ITEM, ImmutableSet.of("cat1", "cat2")); + expected.putAll(BrooklynObjectType.ENTITY, ImmutableSet.of("ent1", "ent2")); + expected.putAll(BrooklynObjectType.LOCATION, ImmutableSet.of("loc1", "loc2")); + expected.putAll(BrooklynObjectType.POLICY, ImmutableSet.of("pol1", "pol2")); + expected.putAll(BrooklynObjectType.ENRICHER, ImmutableSet.of("enricher1", "enricher2")); + expected.putAll(BrooklynObjectType.FEED, ImmutableSet.of("feed1", "feed2")); + assertEquals(deletions, expected); + } + + @Test + public void testLoadFailsIfInvalidDeletionTypeFromYaml() throws Exception { + String contents = + "- deletions:\n"+ + " wrong:\n"+ + " - cat1\n"; + + try { + CompoundTransformer transformer = CompoundTransformerLoader.load(contents); + Asserts.shouldHaveFailedPreviously("transformer="+transformer); + } catch (IllegalStateException e) { + Asserts.expectedFailureContains(e, "Unsupported transform"); + } + } + public static class MyRawDataTransformer implements RawDataTransformer { @Override public String transform(String input) throws Exception { http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/49a4774a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java index ef4e468..2fbd74e 100644 --- a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java @@ -19,12 +19,15 @@ package org.apache.brooklyn.core.mgmt.rebind.transformer; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import java.io.File; import javax.annotation.Nullable; +import org.apache.brooklyn.api.catalog.CatalogItem; import org.apache.brooklyn.api.entity.Entity; import org.apache.brooklyn.api.mgmt.ManagementContext; import org.apache.brooklyn.api.mgmt.ha.HighAvailabilityMode; @@ -44,8 +47,8 @@ import org.apache.brooklyn.core.mgmt.rebind.RebindOptions; import org.apache.brooklyn.core.mgmt.rebind.RebindTestFixtureWithApp; import org.apache.brooklyn.core.mgmt.rebind.RebindTestUtils; import org.apache.brooklyn.core.mgmt.rebind.RecordingRebindExceptionHandler; -import org.apache.brooklyn.core.mgmt.rebind.transformer.CompoundTransformer; import org.apache.brooklyn.core.test.entity.TestApplication; +import org.apache.brooklyn.entity.stock.BasicApplication; import org.apache.brooklyn.util.guava.SerializablePredicate; import org.apache.brooklyn.util.os.Os; import org.slf4j.Logger; @@ -54,10 +57,11 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; +import com.google.common.base.Joiner; import com.google.common.base.Objects; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import com.google.common.collect.Multimap; @SuppressWarnings("serial") public class CompoundTransformerTest extends RebindTestFixtureWithApp { @@ -392,6 +396,37 @@ public class CompoundTransformerTest extends RebindTestFixtureWithApp { assertSingleXmlTransformation(transformer, input, expected); } + @Test + public void testDeleteCatalogItem() throws Exception { + CompoundTransformer transformer = CompoundTransformer.builder() + .deletion(BrooklynObjectType.CATALOG_ITEM, ImmutableList.of("foo:1.2")) + .build(); + + String yaml = Joiner.on("\n").join( + "brooklyn.catalog:", + " items:", + " - id: foo", + " version: 1.2", + " itemType: template", + " item:", + " services:", + " - type: "+BasicApplication.class.getName(), + " - id: bar", + " version: 1.2", + " itemType: template", + " item:", + " services:", + " - type: "+BasicApplication.class.getName()); + + mgmt().getCatalog().addItems(yaml); + CatalogItem<?, ?> origItem = mgmt().getCatalog().getCatalogItem("foo", "1.2"); + assertNotNull(origItem); + + transformAndRebind(transformer); + assertNull(mgmt().getCatalog().getCatalogItem("foo", "1.2")); + assertNotNull(mgmt().getCatalog().getCatalogItem("bar", "1.2")); + } + protected TestApplication transformAndRebind(CompoundTransformer transformer) throws Exception { RebindTestUtils.waitForPersisted(origApp); BrooklynMementoRawData newRawData = transform(origManagementContext, transformer);