METRON-1438 STELLAR: Move shell functions to common from metron-management (ottobackwards) closes apache/metron#920
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c26abbba Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c26abbba Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c26abbba Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: c26abbbaeaeea3551e218cb9aa8ba97b32655958 Parents: 3d3c43c Author: ottobackwards <ottobackwa...@gmail.com> Authored: Fri Feb 2 09:58:59 2018 -0500 Committer: otto <o...@apache.org> Committed: Fri Feb 2 09:58:59 2018 -0500 ---------------------------------------------------------------------- metron-platform/metron-management/README.md | 32 +- .../metron/management/ShellFunctions.java | 302 ------------------- .../metron/management/ShellFunctionsTest.java | 171 ----------- metron-stellar/stellar-common/README.md | 31 ++ metron-stellar/stellar-common/pom.xml | 5 + .../stellar/common/shell/cli/PausableInput.java | 23 +- .../stellar/dsl/functions/ShellFunctions.java | 301 ++++++++++++++++++ .../dsl/functions/ShellFunctionsTest.java | 176 +++++++++++ 8 files changed, 529 insertions(+), 512 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 812583c..bf939c2 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -35,14 +35,13 @@ project. * [Functions](#functions) * [Grok Functions](#grok-functions) * [File Functions](#file-functions) - * [Shell Functions](#shell-functions) * [Configuration Functions](#configuration-functions) * [Parser Functions](#parser-functions) * [Indexing Functions](#indexing-functions) * [Enrichment Functions](#enrichment-functions) * [Threat Triage Functions](#threat-triage-functions) * [Examples](#examples) - * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern) + * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern) * [Manage Stellar Field Transformations](#manage-stellar-field-transformations) * [Manage Stellar Enrichments](#manage-stellar-enrichments) * [Manage Threat Triage Rules](#manage-threat-triage-rules) @@ -132,35 +131,6 @@ The functions are split roughly into a few sections: * path - The path of the file * Returns: true if the file was written and false otherwise. -### Shell Functions - -* `SHELL_EDIT` - * Description: Open an editor (optionally initialized with text) and return whatever is saved from the editor. The editor to use is pulled from `EDITOR` or `VISUAL` environment variable. - * Input: - * string - (Optional) A string whose content is used to initialize the editor. - * Returns: The content that the editor saved after editor exit. -* `SHELL_GET_EXPRESSION` - * Description: Get a stellar expression from a variable - * Input: - * variable - variable name - * Returns: The stellar expression associated with the variable. -* `SHELL_LIST_VARS` - * Description: Return the variables in a tabular form - * Input: - * wrap : Length of string to wrap the columns - * Returns: A tabular representation of the variables. -* `SHELL_MAP2TABLE` - * Description: Take a map and return a table - * Input: - * map - Map - * Returns: The map in table form -* `SHELL_VARS2MAP` - * Description: Take a set of variables and return a map - * Input: - * variables* - variable names to use to create map - * Returns: A map associating the variable name with the stellar expression. - - ### Configuration Functions * `CONFIG_GET` http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java deleted file mode 100644 index afac7f0..0000000 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java +++ /dev/null @@ -1,302 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.management; - -import com.jakewharton.fliptables.FlipTable; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.text.WordUtils; -import org.apache.metron.stellar.common.shell.VariableResult; -import org.apache.metron.stellar.common.shell.cli.PausableInput; -import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; -import org.jboss.aesh.console.Console; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.PrintWriter; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; - -public class ShellFunctions { - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static Map<String, VariableResult> getVariables(Context context) { - return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); - } - - @Stellar( - namespace = "SHELL" - ,name = "MAP2TABLE" - ,description = "Take a map and return a table" - ,params = {"map - Map" - } - ,returns = "The map in table form" - ) - public static class Map2Table extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - if(args.size() < 1) { - return null; - } - Map<Object, Object> map = (Map<Object, Object>) args.get(0); - if(map == null) { - map = new HashMap<>(); - } - String[] headers = {"KEY", "VALUE"}; - String[][] data = new String[map.size()][2]; - int i = 0; - for(Map.Entry<Object, Object> kv : map.entrySet()) { - data[i++] = new String[] {kv.getKey().toString(), kv.getValue().toString()}; - } - return FlipTable.of(headers, data); - } - } - - @Stellar( - namespace = "SHELL" - ,name = "LIST_VARS" - ,description = "Return the variables in a tabular form" - ,params = { - "wrap : Length of string to wrap the columns" - } - ,returns = "A tabular representation of the variables." - ) - public static class ListVars implements StellarFunction { - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - - Map<String, VariableResult> variables = getVariables(context); - String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; - String[][] data = new String[variables.size()][3]; - int wordWrap = -1; - if(args.size() > 0) { - wordWrap = ConversionUtils.convert(args.get(0), Integer.class); - } - int i = 0; - for(Map.Entry<String, VariableResult> kv : variables.entrySet()) { - VariableResult result = kv.getValue(); - data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap) - , toWrappedString(result.getResult(), wordWrap) - , toWrappedString(result.getExpression().get(), wordWrap) - }; - } - return FlipTable.of(headers, data); - } - - private static String toWrappedString(Object o, int wrap) { - String s = "" + o; - if(wrap <= 0) { - return s; - } - return WordUtils.wrap(s, wrap); - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar( - namespace = "SHELL" - ,name = "VARS2MAP" - ,description = "Take a set of variables and return a map" - ,params = {"variables* - variable names to use to create map " - } - ,returns = "A map associating the variable name with the stellar expression." - ) - public static class Var2Map implements StellarFunction { - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - Map<String, VariableResult> variables = getVariables(context); - LinkedHashMap<String, String> ret = new LinkedHashMap<>(); - for(Object arg : args) { - if(arg == null) { - continue; - } - String variable = (String)arg; - VariableResult result = variables.get(variable); - if(result != null && result.getExpression().isPresent()) { - ret.put(variable, result.getExpression().orElseGet(() -> "")); - } - } - return ret; - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar( - namespace = "SHELL" - ,name = "GET_EXPRESSION" - ,description = "Get a stellar expression from a variable" - ,params = {"variable - variable name" - } - ,returns = "The stellar expression associated with the variable." - ) - public static class GetExpression implements StellarFunction { - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - Map<String, VariableResult> variables = getVariables(context); - if(args.size() == 0) { - return null; - } - String variable = (String) args.get(0); - if(variable == null) { - return null; - } - VariableResult result = variables.get(variable); - if(result != null && result.getExpression().isPresent()) { - return result.getExpression().get(); - } - return null; - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - @Stellar( - namespace = "SHELL" - ,name = "EDIT" - ,description = "Open an editor (optionally initialized with text) and return " + - "whatever is saved from the editor. The editor to use is pulled " + - "from `EDITOR` or `VISUAL` environment variable." - ,params = { "string - (Optional) A string whose content is used to initialize the editor." - } - ,returns = "The content that the editor saved after editor exit." - ) - public static class Edit implements StellarFunction { - - private String getEditor() { - // if we have editor in the system properties, it should - // override the env so we check that first - String editor = System.getProperty("EDITOR"); - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = System.getenv().get("EDITOR"); - } - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = System.getenv("VISUAL"); - } - if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { - editor = "/bin/vi"; - } - return editor; - } - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - File outFile = null; - String editor = getEditor(); - try { - outFile = File.createTempFile("stellar_shell", "out"); - if(args.size() > 0) { - String arg = (String)args.get(0); - try(PrintWriter pw = new PrintWriter(outFile)) { - IOUtils.write(arg, pw); - } - } - } catch (IOException e) { - String message = "Unable to create temp file: " + e.getMessage(); - LOG.error(message, e); - throw new IllegalStateException(message, e); - } - Optional<Object> console = context.getCapability(CONSOLE, false); - try { - PausableInput.INSTANCE.pause(); - //shut down the IO for the console - ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath()); - processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); - processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); - processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); - try { - Process p = processBuilder.start(); - // wait for termination. - p.waitFor(); - try (BufferedReader br = new BufferedReader(new FileReader(outFile))) { - String ret = IOUtils.toString(br).trim(); - return ret; - } - } catch (Exception e) { - String message = "Unable to read output: " + e.getMessage(); - LOG.error(message, e); - return null; - } - } finally { - try { - PausableInput.INSTANCE.unpause(); - if(console.isPresent()) { - ((Console)console.get()).pushToInputStream("\b\n"); - } - } catch (IOException e) { - LOG.error("Unable to unpause: {}", e.getMessage(), e); - } - if(outFile.exists()) { - outFile.delete(); - } - } - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java deleted file mode 100644 index 83c2bce..0000000 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.management; - -import com.google.common.collect.ImmutableMap; -import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.stellar.common.shell.VariableResult; -import org.apache.metron.stellar.dsl.Context; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; - -public class ShellFunctionsTest { - - Map<String, VariableResult> variables = ImmutableMap.of( - "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"), - "var2" , VariableResult.withValue("foo"), - "var3" , VariableResult.withValue(null), - "var4" , VariableResult.withExpression(null, "blah") - ); - - Context context = new Context.Builder() - .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) - .build(); -/** -ââââââââââââ¤ââââââââ¤âââââââââââââ -â VARIABLE â VALUE â EXPRESSION â -â âââââââââââªââââââââªââââââââââââ⣠-â foo â 2.0 â 1 + 1 â -ââââââââââââ§ââââââââ§âââââââââââââ - **/ - @Multiline - static String expectedListWithFoo; - - @Test - public void testListVarsWithVars() { - Map<String, VariableResult> variables = ImmutableMap.of( - "foo", VariableResult.withExpression(2.0, "1 + 1")); - - Context context = new Context.Builder() - .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) - .build(); - Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); - Assert.assertEquals(expectedListWithFoo, out); - } - -/** -ââââââââââââ¤ââââââââ¤âââââââââââââ -â VARIABLE â VALUE â EXPRESSION â -â âââââââââââ§ââââââââ§ââââââââââââ⣠-â (empty) â -âââââââââââââââââââââââââââââââââ - **/ - @Multiline - static String expectedEmptyList; - - @Test - public void testListVarsWithoutVars() { - Context context = new Context.Builder() - .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>()) - .build(); - Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); - Assert.assertEquals(expectedEmptyList, out); - } -/** -ââââââââââ¤ââââââââ -â KEY â VALUE â -â âââââââââªâââââââ⣠-â field1 â val1 â -ââââââââââ¼âââââââ⢠-â field2 â val2 â -ââââââââââ§ââââââââ - **/ - @Multiline - static String expectedMap2Table; - - @Test - public void testMap2Table() { - Map<String, Object> variables = ImmutableMap.of("map_field", ImmutableMap.of("field1", "val1", "field2", "val2")); - Context context = Context.EMPTY_CONTEXT(); - Object out = run("SHELL_MAP2TABLE(map_field)", variables, context); - Assert.assertEquals(expectedMap2Table, out); - } - /** -âââââââ¤ââââââââ -â KEY â VALUE â -â ââââââ§âââââââ⣠-â (empty) â -âââââââââââââââ - **/ - @Multiline - static String expectedMap2TableNullInput; - - @Test - public void testMap2TableNullInput() { - Map<String,Object> variables = new HashMap<String,Object>(){{ - put("map_field",null); - }}; - Context context = Context.EMPTY_CONTEXT(); - Object out = run("SHELL_MAP2TABLE(map_field)", variables, context); - Assert.assertEquals(expectedMap2TableNullInput, out); - } - - @Test - public void testMap2TableInsufficientArgs() { - Map<String, Object> variables = new HashMap<>(); - Context context = Context.EMPTY_CONTEXT(); - Object out = run("SHELL_MAP2TABLE()", variables, context); - Assert.assertNull(out); - } - - @Test - public void testVars2Map() { - Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context); - Assert.assertTrue(out instanceof Map); - Map<String, String> mapOut = (Map<String, String>)out; - //second one is null, so we don't want it there. - Assert.assertEquals(1, mapOut.size()); - Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1")); - } - - @Test - public void testVars2MapEmpty() { - Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context); - Map<String, String> mapOut = (Map<String, String>)out; - Assert.assertEquals(0, mapOut.size()); - } - - @Test - public void testGetExpression() { - Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context); - Assert.assertTrue(out instanceof String); - String expression = (String)out; - //second one is null, so we don't want it there. - Assert.assertEquals("TO_UPPER('casey')", expression); - } - - @Test - public void testGetExpressionEmpty() { - Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context); - Assert.assertNull(out ); - } - - @Test - public void testEdit() throws Exception { - System.getProperties().put("EDITOR", "/bin/cat"); - Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context); - Assert.assertEquals("FOO", out); - } - -} http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index a2f1242..078799e 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -243,6 +243,11 @@ Where: | [ `SET_MERGE`](#set_merge) | | [ `SET_REMOVE`](#set_remove) | | [ `SIN`](#sin) | +| [ `SHELL_EDIT`](#shell_edit) | +| [ `SHELL_GET_EXPRESSION`](#shell_get_expression) | +| [ `SHELL_LIST_VARS`](#shell_list_vars) | +| [ `SHELL_MAP2TABLE`](#shell_map2table) | +| [ `SHELL_VARS2MAP`](#shell_vars2map) | | [ `SPLIT`](#split) | | [ `SQRT`](#sqrt) | | [ `STARTS_WITH`](#starts_with) | @@ -920,6 +925,32 @@ Where: * o - object to add to set * Returns: A Set +### `SHELL_EDIT` + * Description: Open an editor (optionally initialized with text) and return whatever is saved from the editor. The editor to use is pulled from `EDITOR` or `VISUAL` environment variable. + * Input: + * string - (Optional) A string whose content is used to initialize the editor. + * Returns: The content that the editor saved after editor exit. +### `SHELL_GET_EXPRESSION` + * Description: Get a stellar expression from a variable + * Input: + * variable - variable name + * Returns: The stellar expression associated with the variable. +### `SHELL_LIST_VARS` + * Description: Return the variables in a tabular form + * Input: + * wrap : Length of string to wrap the columns + * Returns: A tabular representation of the variables. +### `SHELL_MAP2TABLE` + * Description: Take a map and return a table + * Input: + * map - Map + * Returns: The map in table form +### `SHELL_VARS2MAP` + * Description: Take a set of variables and return a map + * Input: + * variables* - variable names to use to create map + * Returns: A map associating the variable name with the stellar expression. + ### `SIN` * Description: Returns the sine of a number. * Input: http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index b43fcc1..6b07e68 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -203,6 +203,11 @@ <version>${global_kryo_serializers_version}</version> </dependency> <dependency> + <groupId>com.jakewharton.fliptables</groupId> + <artifactId>fliptables</artifactId> + <version>1.0.2</version> + </dependency> + <dependency> <!-- junit dependency added with default scope to allow inclusion of StellarProcessorUtils in main jar. It is excluded from the uber-jar. --> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java index fad0115..c72d66f 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java @@ -21,6 +21,7 @@ package org.apache.metron.stellar.common.shell.cli; import java.io.IOException; import java.io.InputStream; +import java.util.concurrent.atomic.AtomicBoolean; /** * An input stream which mirrors System.in, but allows you to 'pause' and 'unpause' it. @@ -36,8 +37,8 @@ import java.io.InputStream; * */ public class PausableInput extends InputStream { - InputStream in = System.in; - boolean paused = false; + private InputStream in = System.in; + private AtomicBoolean paused = new AtomicBoolean(false); private PausableInput() { super(); } @@ -46,7 +47,7 @@ public class PausableInput extends InputStream { * Stop mirroring stdin */ public void pause() { - paused = true; + paused.set(true); } /** @@ -54,8 +55,7 @@ public class PausableInput extends InputStream { * @throws IOException */ public void unpause() throws IOException { - in.read(new byte[in.available()]); - paused = false; + paused.set(false); } public final static PausableInput INSTANCE = new PausableInput(); @@ -76,7 +76,14 @@ public class PausableInput extends InputStream { */ @Override public int read() throws IOException { - + if(paused.get()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 0; + } return in.read(); } @@ -116,7 +123,7 @@ public class PausableInput extends InputStream { @Override public int read(byte[] b) throws IOException { - if(paused) { + if(paused.get()) { try { Thread.sleep(1000); } catch (InterruptedException e) { @@ -187,7 +194,7 @@ public class PausableInput extends InputStream { */ @Override public int read(byte[] b, int off, int len) throws IOException { - if(paused) { + if(paused.get()) { try { Thread.sleep(1000); } catch (InterruptedException e) { http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java new file mode 100644 index 0000000..1df4a51 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java @@ -0,0 +1,301 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.dsl.functions; + +import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; + +import com.jakewharton.fliptables.FlipTable; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.metron.stellar.common.shell.VariableResult; +import org.apache.metron.stellar.common.shell.cli.PausableInput; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.jboss.aesh.console.Console; +import org.slf4j.LoggerFactory; + +public class ShellFunctions { + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @SuppressWarnings("unchecked") + private static Map<String, VariableResult> getVariables(Context context) { + return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); + } + + @Stellar( + namespace = "SHELL" + ,name = "MAP2TABLE" + ,description = "Take a map and return a table" + ,params = {"map - Map" + } + ,returns = "The map in table form" + ) + public static class Map2Table extends BaseStellarFunction { + + @Override + @SuppressWarnings("unchecked") + public Object apply(List<Object> args) { + if (args.size() < 1) { + return null; + } + Map<Object, Object> map = (Map<Object, Object>) args.get(0); + if (map == null) { + map = new HashMap<>(); + } + String[] headers = {"KEY", "VALUE"}; + String[][] data = new String[map.size()][2]; + int i = 0; + for (Map.Entry<Object, Object> kv : map.entrySet()) { + data[i++] = new String[]{kv.getKey().toString(), kv.getValue().toString()}; + } + return FlipTable.of(headers, data); + } + } + + @Stellar( + namespace = "SHELL" + ,name = "LIST_VARS" + ,description = "Return the variables in a tabular form" + ,params = { + "wrap : Length of string to wrap the columns" + } + ,returns = "A tabular representation of the variables." + ) + public static class ListVars implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + Map<String, VariableResult> variables = getVariables(context); + String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; + String[][] data = new String[variables.size()][3]; + int wordWrap = -1; + if (args.size() > 0) { + wordWrap = ConversionUtils.convert(args.get(0), Integer.class); + } + int i = 0; + for (Map.Entry<String, VariableResult> kv : variables.entrySet()) { + VariableResult result = kv.getValue(); + data[i++] = new String[]{toWrappedString(kv.getKey(), wordWrap), + toWrappedString(result.getResult(), wordWrap), + toWrappedString(result.getExpression().get(), wordWrap)}; + } + return FlipTable.of(headers, data); + } + + private static String toWrappedString(Object o, int wrap) { + String s = "" + o; + if(wrap <= 0) { + return s; + } + return WordUtils.wrap(s, wrap); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Stellar( + namespace = "SHELL" + ,name = "VARS2MAP" + ,description = "Take a set of variables and return a map" + ,params = {"variables* - variable names to use to create map " + } + ,returns = "A map associating the variable name with the stellar expression." + ) + public static class Var2Map implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + Map<String, VariableResult> variables = getVariables(context); + LinkedHashMap<String, String> ret = new LinkedHashMap<>(); + for (Object arg : args) { + if (arg == null) { + continue; + } + String variable = (String) arg; + VariableResult result = variables.get(variable); + if (result != null && result.getExpression().isPresent()) { + ret.put(variable, result.getExpression().orElseGet(() -> "")); + } + } + return ret; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Stellar( + namespace = "SHELL" + ,name = "GET_EXPRESSION" + ,description = "Get a stellar expression from a variable" + ,params = {"variable - variable name" + } + ,returns = "The stellar expression associated with the variable." + ) + public static class GetExpression implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + Map<String, VariableResult> variables = getVariables(context); + if (args.size() == 0) { + return null; + } + String variable = (String) args.get(0); + if (variable == null) { + return null; + } + VariableResult result = variables.get(variable); + if (result != null && result.getExpression().isPresent()) { + return result.getExpression().get(); + } + return null; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Stellar( + namespace = "SHELL" + ,name = "EDIT" + ,description = "Open an editor (optionally initialized with text) and return " + + "whatever is saved from the editor. The editor to use is pulled " + + "from `EDITOR` or `VISUAL` environment variable." + ,params = { "string - (Optional) A string whose content is used to initialize the editor." + } + ,returns = "The content that the editor saved after editor exit." + ) + public static class Edit implements StellarFunction { + + private String getEditor() { + // if we have editor in the system properties, it should + // override the env so we check that first + String editor = System.getProperty("EDITOR"); + if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = System.getenv().get("EDITOR"); + } + if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = System.getenv("VISUAL"); + } + if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) { + editor = "/bin/vi"; + } + return editor; + } + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + File outFile = null; + String editor = getEditor(); + try { + outFile = File.createTempFile("stellar_shell", "out"); + if (args.size() > 0) { + String arg = (String) args.get(0); + try (PrintWriter pw = new PrintWriter(outFile)) { + IOUtils.write(arg, pw); + } + } + } catch (IOException e) { + String message = "Unable to create temp file: " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + Optional<Object> console = context.getCapability(CONSOLE, false); + try { + PausableInput.INSTANCE.pause(); + //shut down the IO for the console + ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath()); + processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); + processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); + try { + Process p = processBuilder.start(); + // wait for termination. + p.waitFor(); + try (BufferedReader br = new BufferedReader(new FileReader(outFile))) { + String ret = IOUtils.toString(br).trim(); + return ret; + } + } catch (Exception e) { + String message = "Unable to read output: " + e.getMessage(); + LOG.error(message, e); + return null; + } + } finally { + try { + PausableInput.INSTANCE.unpause(); + if (console.isPresent()) { + ((Console) console.get()).pushToInputStream("\b\n"); + } + } catch (IOException e) { + LOG.error("Unable to unpause: {}", e.getMessage(), e); + } + if (outFile.exists()) { + outFile.delete(); + } + } + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java new file mode 100644 index 0000000..354e0c3 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableMap; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.stellar.common.shell.VariableResult; +import org.apache.metron.stellar.common.shell.cli.PausableInput; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Context.Capabilities; +import org.jboss.aesh.console.Console; +import org.jboss.aesh.console.settings.Settings; +import org.jboss.aesh.console.settings.SettingsBuilder; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; + +public class ShellFunctionsTest { + + Map<String, VariableResult> variables = ImmutableMap.of( + "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"), + "var2" , VariableResult.withValue("foo"), + "var3" , VariableResult.withValue(null), + "var4" , VariableResult.withExpression(null, "blah") + ); + + Context context = new Context.Builder() + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables).build(); + +/** +ââââââââââââ¤ââââââââ¤âââââââââââââ +â VARIABLE â VALUE â EXPRESSION â +â âââââââââââªââââââââªââââââââââââ⣠+â foo â 2.0 â 1 + 1 â +ââââââââââââ§ââââââââ§âââââââââââââ + **/ + @Multiline + static String expectedListWithFoo; + + @Test + public void testListVarsWithVars() { + Map<String, VariableResult> variables = ImmutableMap.of( + "foo", VariableResult.withExpression(2.0, "1 + 1")); + + Context context = new Context.Builder() + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) + .build(); + Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); + Assert.assertEquals(expectedListWithFoo, out); + } + +/** +ââââââââââââ¤ââââââââ¤âââââââââââââ +â VARIABLE â VALUE â EXPRESSION â +â âââââââââââ§ââââââââ§ââââââââââââ⣠+â (empty) â +âââââââââââââââââââââââââââââââââ + **/ + @Multiline + static String expectedEmptyList; + + @Test + public void testListVarsWithoutVars() { + Context context = new Context.Builder() + .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>()) + .build(); + Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); + Assert.assertEquals(expectedEmptyList, out); + } +/** +ââââââââââ¤ââââââââ +â KEY â VALUE â +â âââââââââªâââââââ⣠+â field1 â val1 â +ââââââââââ¼âââââââ⢠+â field2 â val2 â +ââââââââââ§ââââââââ + **/ + @Multiline + static String expectedMap2Table; + + @Test + public void testMap2Table() { + Map<String, Object> variables = ImmutableMap.of("map_field", ImmutableMap.of("field1", "val1", "field2", "val2")); + Context context = Context.EMPTY_CONTEXT(); + Object out = run("SHELL_MAP2TABLE(map_field)", variables, context); + Assert.assertEquals(expectedMap2Table, out); + } + /** +âââââââ¤ââââââââ +â KEY â VALUE â +â ââââââ§âââââââ⣠+â (empty) â +âââââââââââââââ + **/ + @Multiline + static String expectedMap2TableNullInput; + + @Test + public void testMap2TableNullInput() { + Map<String,Object> variables = new HashMap<String,Object>(){{ + put("map_field",null); + }}; + Context context = Context.EMPTY_CONTEXT(); + Object out = run("SHELL_MAP2TABLE(map_field)", variables, context); + Assert.assertEquals(expectedMap2TableNullInput, out); + } + + @Test + public void testMap2TableInsufficientArgs() { + Map<String, Object> variables = new HashMap<>(); + Context context = Context.EMPTY_CONTEXT(); + Object out = run("SHELL_MAP2TABLE()", variables, context); + Assert.assertNull(out); + } + + @Test + public void testVars2Map() { + Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context); + Assert.assertTrue(out instanceof Map); + Map<String, String> mapOut = (Map<String, String>)out; + //second one is null, so we don't want it there. + Assert.assertEquals(1, mapOut.size()); + Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1")); + } + + @Test + public void testVars2MapEmpty() { + Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context); + Map<String, String> mapOut = (Map<String, String>)out; + Assert.assertEquals(0, mapOut.size()); + } + + @Test + public void testGetExpression() { + Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context); + Assert.assertTrue(out instanceof String); + String expression = (String)out; + //second one is null, so we don't want it there. + Assert.assertEquals("TO_UPPER('casey')", expression); + } + + @Test + public void testGetExpressionEmpty() { + Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context); + Assert.assertNull(out ); + } + + @Test + public void testEdit() throws Exception { + System.getProperties().put("EDITOR", "/bin/cat"); + Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context); + Assert.assertEquals("FOO", out); + } + +}
