METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85cd21aa Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85cd21aa Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85cd21aa Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 85cd21aa0f5045184c168248dc2b81c1cfd41ddd Parents: b9461e7 Author: mmiklavc <michael.miklav...@gmail.com> Authored: Tue Nov 6 18:09:56 2018 -0700 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Tue Nov 6 18:09:56 2018 -0700 ---------------------------------------------------------------------- .../ElasticsearchSearchIntegrationTest.java | 1 - .../metron/stellar/dsl/StellarFunction.java | 9 +- .../metron/stellar/dsl/StellarFunctions.java | 5 + .../resolver/BaseFunctionResolver.java | 44 +++++ .../functions/resolver/FunctionResolver.java | 14 +- .../stellar/dsl/functions/BasicStellarTest.java | 20 ++- .../resolver/BaseFunctionResolverTest.java | 169 +++++++++++++++++++ 7 files changed, 251 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 1d2d48e..8187468 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java index efdd185..4fabfaf 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java @@ -17,10 +17,17 @@ */ package org.apache.metron.stellar.dsl; +import java.io.Closeable; +import java.io.IOException; import java.util.List; -public interface StellarFunction { +public interface StellarFunction extends Closeable { Object apply(List<Object> args, Context context) throws ParseException; void initialize(Context context); boolean isInitialized(); + + @Override + default void close() throws IOException { + + } } http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java index dfec90e..73df82f 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java @@ -18,6 +18,7 @@ package org.apache.metron.stellar.dsl; +import java.io.IOException; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver; @@ -30,4 +31,8 @@ public class StellarFunctions { public static void initialize(Context context) { SingletonFunctionResolver.getInstance().initialize(context); } + + public static void close() throws IOException { + SingletonFunctionResolver.getInstance().close(); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java index aeed9d9..38a32d1 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java @@ -23,6 +23,7 @@ import static java.lang.String.format; import com.google.common.base.Joiner; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import java.io.IOException; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.util.Arrays; @@ -58,9 +59,15 @@ public abstract class BaseFunctionResolver implements FunctionResolver, Serializ */ protected Context context; + /** + * Indicates if closed has been called on this resolver. + */ + private boolean closed; + public BaseFunctionResolver() { // memoize provides lazy initialization and thread-safety (the ugly cast is necessary for serialization) functions = Suppliers.memoize((Supplier<Map<String, StellarFunctionInfo>> & Serializable) this::resolveFunctions); + closed = false; } /** @@ -95,6 +102,43 @@ public abstract class BaseFunctionResolver implements FunctionResolver, Serializ } /** + * Makes an attempt to close all Stellar functions. Calling close multiple times has no effect. + * @throws IOException Catches all exceptions and summarizes them. + */ + @Override + public void close() throws IOException { + if (!closed) { + LOG.info("Calling close() on Stellar functions."); + Map<String, Throwable> errors = new HashMap<>(); + for (StellarFunctionInfo info : getFunctionInfo()) { + try { + info.getFunction().close(); + } catch (Throwable t) { + errors.put(info.getName(), t); + } + } + if (!errors.isEmpty()) { + StringBuilder sb = new StringBuilder(); + sb.append("Unable to close Stellar functions:"); + for (Map.Entry<String, Throwable> e : errors.entrySet()) { + Throwable throwable = e.getValue(); + String eText = String + .format("Exception - Function: %s; Message: %s; Cause: %s", e.getKey(), + throwable.getMessage(), + throwable.getCause()); + sb.append(System.lineSeparator()); + sb.append(eText); + } + closed = true; + throw new IOException(sb.toString()); + } + closed = true; + } else { + LOG.info("close() already called on Stellar functions - skipping."); + } + } + + /** * Resolves a function by name. * @param functionName The name of the function to resolve. * @return The executable StellarFunction. http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java index 5acb42c..4047586 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java @@ -17,16 +17,17 @@ */ package org.apache.metron.stellar.dsl.functions.resolver; +import java.io.Closeable; +import java.io.IOException; +import java.util.function.Function; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.stellar.dsl.StellarFunctionInfo; -import java.util.function.Function; - /** * Responsible for function resolution in Stellar. */ -public interface FunctionResolver extends Function<String, StellarFunction> { +public interface FunctionResolver extends Function<String, StellarFunction>, Closeable { /** * Provides metadata about each Stellar function that is resolvable. @@ -43,4 +44,11 @@ public interface FunctionResolver extends Function<String, StellarFunction> { * @param context Context used to initialize. */ void initialize(Context context); + + /** + * Perform any cleanup necessary for the loaded Stellar functions. + */ + @Override + default void close() throws IOException {} + } http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index dec05a8..79f97bc 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -18,9 +18,20 @@ package org.apache.metron.stellar.dsl.functions; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate; + import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; @@ -37,12 +48,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.*; - -import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; -import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate; -import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; - @SuppressWarnings("ALL") public class BasicStellarTest { @@ -70,6 +75,7 @@ public class BasicStellarTest { public boolean isInitialized() { return true; } + } @Stellar( @@ -96,6 +102,7 @@ public class BasicStellarTest { public boolean isInitialized() { return true; } + } @Test @@ -1000,4 +1007,5 @@ public class BasicStellarTest { checkFalsey("{}"); checkFalsey("LIST_ADD([])"); } + } http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java new file mode 100644 index 0000000..47cbda3 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions.resolver; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class BaseFunctionResolverTest { + + public static class TestResolver extends BaseFunctionResolver { + + Set<Class<? extends StellarFunction>> classesToResolve = new HashSet<>(); + + @Override + public Set<Class<? extends StellarFunction>> resolvables() { + return classesToResolve; + } + + /** + * Will attempt to resolve any Stellar functions defined within the specified class. + * @param clazz The class which may contain a Stellar function. + */ + public TestResolver withClass(Class<? extends StellarFunction> clazz) { + this.classesToResolve.add(clazz); + return this; + } + } + + /** + * Often imitated, never duplicated. + */ + @Stellar(namespace = "namespace", name = "afunction", description = "description", returns = "returns", params = { + "param1"}) + private static class IAmAFunction extends BaseStellarFunction { + + public static int closeCallCount; + public static boolean throwException = false; // init here bc of reflection in resolver. + + public IAmAFunction() { + closeCallCount = 0; + } + + @Override + public Object apply(List<Object> args) { + return null; + } + + @Override + public void close() throws IOException { + closeCallCount++; + if (throwException) { + Throwable cause = new Throwable("Some nasty nasty cause."); + throw new IOException("Bad things happened", cause); + } + } + } + + /** + * Scratch that. I was wrong. + */ + @Stellar(namespace = "namespace", name = "anotherfunction", description = "description", returns = "returns", params = { + "param1"}) + private static class IAmAnotherFunction extends BaseStellarFunction { + + public static int closeCallCount; + public static boolean throwException = false; // init here bc of reflection in resolver. + + public IAmAnotherFunction() { + closeCallCount = 0; + } + + @Override + public Object apply(List<Object> args) { + return null; + } + + @Override + public void close() throws IOException { + closeCallCount++; + if (throwException) { + throw new NullPointerException("A most annoying exception."); + } + } + } + + private TestResolver resolver; + + @Before + public void setup() { + resolver = new TestResolver(); + IAmAFunction.throwException = false; + IAmAnotherFunction.throwException = false; + } + + @Test + public void close_calls_all_loaded_function_close_methods() throws IOException { + resolver.withClass(IAmAFunction.class); + resolver.withClass(IAmAnotherFunction.class); + resolver.close(); + assertThat(IAmAFunction.closeCallCount, equalTo(1)); + assertThat(IAmAnotherFunction.closeCallCount, equalTo(1)); + } + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void close_collects_all_exceptions_thrown_on_loaded_function_close_methods() + throws IOException { + IAmAFunction.throwException = true; + IAmAnotherFunction.throwException = true; + resolver.withClass(IAmAFunction.class); + resolver.withClass(IAmAnotherFunction.class); + exception.expect(IOException.class); + resolver.close(); + } + + @Test + public void close_only_throws_exceptions_on_first_invocation() + throws IOException { + IAmAFunction.throwException = true; + IAmAnotherFunction.throwException = true; + resolver.withClass(IAmAFunction.class); + resolver.withClass(IAmAnotherFunction.class); + try { + resolver.close(); + Assert.fail("Should have thrown an exception."); + } catch (IOException e) { + // intentionally empty + } + assertThat(IAmAFunction.closeCallCount, equalTo(1)); + assertThat(IAmAnotherFunction.closeCallCount, equalTo(1)); + // should not throw exceptions or call any function's close again. + resolver.close(); + resolver.close(); + resolver.close(); + assertThat(IAmAFunction.closeCallCount, equalTo(1)); + assertThat(IAmAnotherFunction.closeCallCount, equalTo(1)); + } +}
