Repository: metron Updated Branches: refs/heads/master 9355a0465 -> e206f2508
METRON-984 Create STELLAR Decoding Functions (ottobackwards) closes apache/metron#642 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e206f250 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e206f250 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e206f250 Branch: refs/heads/master Commit: e206f2508ef7e7d798510df76ccfeb38b9530e89 Parents: 9355a04 Author: ottobackwards <ottobackwa...@gmail.com> Authored: Thu Jul 27 23:37:28 2017 -0400 Committer: otto <o...@apache.org> Committed: Thu Jul 27 23:37:28 2017 -0400 ---------------------------------------------------------------------- metron-stellar/stellar-common/README.md | 52 +++++- metron-stellar/stellar-common/pom.xml | 5 + .../stellar/common/encoding/Encodings.java | 158 +++++++++++++++++++ .../dsl/functions/EncodingFunctions.java | 157 ++++++++++++++++++ .../stellar/common/encoding/EncodingsTest.java | 104 ++++++++++++ .../stellar/dsl/functions/BasicStellarTest.java | 2 - .../dsl/functions/EncodingFunctionsTest.java | 125 +++++++++++++++ 7 files changed, 594 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 0ffb096..2b5d4b6 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -116,15 +116,17 @@ In the core language functions, we support basic functional programming primitiv | [ `DAY_OF_MONTH`](#day_of_month) | | [ `DAY_OF_WEEK`](#day_of_week) | | [ `DAY_OF_YEAR`](#day_of_year) | +| [ `DECODE`](#decode) | | [ `DOMAIN_REMOVE_SUBDOMAINS`](#domain_remove_subdomains) | | [ `DOMAIN_REMOVE_TLD`](#domain_remove_tld) | | [ `DOMAIN_TO_TLD`](#domain_to_tld) | +| [ `ENCODE`](#encode) | | [ `ENDS_WITH`](#ends_with) | | [ `ENRICHMENT_EXISTS`](#enrichment_exists) | | [ `ENRICHMENT_GET`](#enrichment_get) | | [ `FILL_LEFT`](#fill_left) | | [ `FILL_RIGHT`](#fill_right) | -| [ `FILTER`](#filter) | +| [ `FILTER`](#filter) | | [ `FORMAT`](#format) | | [ `HLLP_CARDINALITY`](../../metron-analytics/metron-statistics#hllp_cardinality) | | [ `HLLP_INIT`](../../metron-analytics/metron-statistics#hllp_init) | @@ -134,8 +136,10 @@ In the core language functions, we support basic functional programming primitiv | [ `GET`](#get) | | [ `GET_FIRST`](#get_first) | | [ `GET_LAST`](#get_last) | +| [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) | | [ `IN_SUBNET`](#in_subnet) | | [ `IS_DATE`](#is_date) | +| [ `IS_ENCODING`](#is_encoding) | | [ `IS_DOMAIN`](#is_domain) | | [ `IS_EMAIL`](#is_email) | | [ `IS_EMPTY`](#is_empty) | @@ -148,10 +152,10 @@ In the core language functions, we support basic functional programming primitiv | [ `KAFKA_PUT`](#kafka_put) | | [ `KAFKA_TAIL`](#kafka_tail) | | [ `LENGTH`](#length) | -| [ `LIST_ADD`](#list_add) | +| [ `LIST_ADD`](#list_add) | | [ `MAAS_GET_ENDPOINT`](#maas_get_endpoint) | | [ `MAAS_MODEL_APPLY`](#maas_model_apply) | -| [ `MAP`](#map) | +| [ `MAP`](#map) | | [ `MAP_EXISTS`](#map_exists) | | [ `MONTH`](#month) | | [ `PREPEND_IF_MISSING`](#prepend_if_missing) | @@ -159,7 +163,7 @@ In the core language functions, we support basic functional programming primitiv | [ `PROFILE_FIXED`](#profile_fixed) | | [ `PROFILE_WINDOW`](#profile_window) | | [ `PROTOCOL_TO_NAME`](#protocol_to_name) | -| [ `REDUCE`](#reduce) | +| [ `REDUCE`](#reduce) | | [ `REGEXP_MATCH`](#regexp_match) | | [ `REGEXP_GROUP_VAL`](#regexp_group_val) | | [ `SPLIT`](#split) | @@ -277,6 +281,19 @@ In the core language functions, we support basic functional programming primitiv * dateTime - The datetime as a long representing the milliseconds since unix epoch * Returns: The day number within the year. +### `DECODE` + * Description: Decodes the passed string with the provided encoding, which + must be one of the encodings returned from [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) + * Input: + * string - the string to decode + * encoding - the encoding to use, must be one of encodings returned from [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) + * verify - (optional), true or false to determine if string should be verified as being + encoded with the passed encoding + * Returns: + * The decoded string on success + * The original string the string cannot be decoded + * null on usage error + ### `DOMAIN_REMOVE_SUBDOMAINS` * Description: Removes the subdomains from a domain. * Input: @@ -295,6 +312,16 @@ In the core language functions, we support basic functional programming primitiv * domain - Fully qualified domain name * Returns: The TLD of the domain. (for example, DOMAIN_TO_TLD('mail.yahoo.co.uk') yields 'co.uk') +### `ENCODE` + * Description: Encodes the passed string with the provided encoding, which + must be one of the encodings returned from [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) + * Input: + * string - the string to encode + * encoding - the encoding to use, must be one of encodings returned from [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) + * Returns: + * The encoded string on success + * null on error + ### `ENDS_WITH` * Description: Determines whether a string ends with a specified suffix * Input: @@ -405,9 +432,16 @@ In the core language functions, we support basic functional programming primitiv ### `IS_EMPTY` * Description: Returns true if string or collection is empty or null and false if otherwise. * Input: - * input - Object of string or collection type (for example, list) + * Object of string or collection type (for example, list) * Returns: True if the string or collection is empty or null and false if otherwise. +### `IS_ENCODING` + * Description: Returns true if the passed string is encoded in one of the supported encodings and false if otherwise. + * Input: + * string - The string to test + * encoding - The name of the encoding as string. See [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings). + * Returns: True if the passed string is encoded in one of the supported encodings and false if otherwise. + ### `IS_INTEGER` * Description: Determines whether or not an object is an integer. * Input: @@ -477,6 +511,10 @@ In the core language functions, we support basic functional programming primitiv * element - Element to add to list * Returns: Resulting list with the item added at the end. +### `GET_SUPPORTED_ENCODINGS` + * Description: Returns a list of the encodings that are currently supported. + * Returns: A List of String + ### `MAAS_GET_ENDPOINT` * Description: Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters. * Input: @@ -914,8 +952,8 @@ This command lists all functions resolvable in the Stellar environment. Stellar ``` [Stellar]>>> %functions BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, -DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, GET, GET_FIRST, -GET_LAST, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, +DECODE, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, GET, GET_FIRST, +GET_LAST, GET_ENCODINGS_LIST, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_ENCODING, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP_EXISTS, MAP_GET, MONTH, PROTOCOL_TO_NAME, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index 11008b9..2f4cb6e 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -92,6 +92,11 @@ <version>4.1</version> </dependency> <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.10</version> + </dependency> + <dependency> <groupId>commons-validator</groupId> <artifactId>commons-validator</artifactId> <version>1.6</version> http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java new file mode 100644 index 0000000..aceda1f --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/encoding/Encodings.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.encoding; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base32; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.BinaryCodec; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang.StringUtils; + +/** + * Endodings utility enum for Stellar supported encodings. + */ +public enum Encodings { + BASE32((possible) -> new Base32().isInAlphabet(possible), + (possible) -> new String(new Base32().decode(possible), StandardCharsets.UTF_8), + (possible) -> new String( new Base32().encode(possible.getBytes(StandardCharsets.UTF_8)),StandardCharsets.UTF_8)), + BASE32HEX((possible) -> new Base32(true).isInAlphabet(possible), + (possible) -> new String(new Base32(true).decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8), + (possible) -> new String(new Base32(true).encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)), + BASE64((possible) -> Base64.isBase64(possible), + (possible) -> new String(new Base64().decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8), + (possible) -> new String(new Base64().encode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)), + BINARY((possible) -> { + for (byte b : possible.getBytes(StandardCharsets.UTF_8)) { + if ((b != 48 && b != 49)) { + return false; + } + } + return true; + }, + (possible) -> { + String str = new String(BinaryCodec.fromAscii(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + if (StringUtils.isEmpty(str.trim())) { + return possible; + } + return str; + }, + (possible) -> BinaryCodec.toAsciiString(possible.getBytes(StandardCharsets.UTF_8))), + HEX((possible) -> { + try { + Hex hex = new Hex(StandardCharsets.UTF_8); + hex.decode(possible.getBytes(StandardCharsets.UTF_8)); + return true; + } catch (DecoderException e) { + return false; + } + }, + (possible) -> { + try { + Hex hex = new Hex(StandardCharsets.UTF_8); + return new String(hex.decode(possible.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } catch (DecoderException e) { + return possible; + } + }, + (possible) -> new String(new Hex(StandardCharsets.UTF_8).encode(possible.getBytes(StandardCharsets.UTF_8)),StandardCharsets.UTF_8)); + + Predicate<String> is; + Function<String, String> decode; + Function<String, String> encode; + + /** + * Create a specialed Endodings enum member. + * + * @param is function for detecting + * @param decode funtion for decoding + */ + Encodings(Predicate<String> is, Function<String, String> decode, Function<String,String> encode) { + this.is = is; + this.decode = decode; + this.encode = encode; + } + + public static final List<String> SUPPORTED_LIST = new ArrayList<>(Arrays.asList(BASE32.name(), + BASE32HEX.name(), BASE64.name(), BINARY.name() , HEX.name())); + + /** + * Determines if the passed String is encoded in this encoding. + * A given String may be compatible with the encoding, but not actually encoded. + * + * @param possible the String to test + * @return true or false + */ + public boolean is(String possible) { + return is.test(possible); + } + + /** + * Attempts to decode a given String without verification. + * Any failure or compatibility issues will result in the original + * String being returned. + * + * @param encoded the String to decode + * @return The String decoded, or the original String + */ + public String decode(String encoded) { + return decode(encoded, false); + } + + /** + * Attempts to decode a given String, optionally verifying first. + * Any failure or compatibility issues will result in the original + * String being returned. + * + * @param encoded the String to decode + * @param verify flag to perform verification + * @return The String decoded, or the original String + */ + public String decode(String encoded, boolean verify) { + if (verify) { + if (is.test(encoded)) { + return decode.apply(encoded); + } else { + return encoded; + } + } + return decode.apply(encoded); + } + + /** + * Encodes the given String + * @param toEncode + * @return an encoded String + */ + public String encode(String toEncode) { + return encode.apply(toEncode); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java new file mode 100644 index 0000000..41783c2 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/EncodingFunctions.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import java.util.List; +import org.apache.commons.lang.StringUtils; +import org.apache.metron.stellar.common.encoding.Encodings; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; + +/** + * Support for Stellar Functions based on the Apache Commons Codec library + * http://commons.apache.org/proper/commons-codec/index.html + */ +public class EncodingFunctions { + + @Stellar(name = "GET_SUPPORTED_ENCODINGS", + description = "Returns a list of the encodings that are currently supported as a list", + params = {}, + returns = "A list of supported encodings" + ) + public static class GetSupportedEncodings extends BaseStellarFunction { + + @Override + public Object apply(List<Object> list) { + return Encodings.SUPPORTED_LIST; + } + } + + @Stellar(name = "IS_ENCODING", + description = "Returns if the passed string is encoded in one of the supported encodings", + params = {"string - the string to test", + "encoding - the encoding to test, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS" + }, + returns = "true if it is encoded, false if not" + ) + public static class IsEncoding extends BaseStellarFunction { + + @Override + public Object apply(List<Object> list) { + if (list.size() < 2) { + throw new IllegalStateException( + "IS_ENCODING expects two args: [string, encoding] where encoding is one from " + + "the supported list"); + } + String str = (String) list.get(0); + String encoding = (String) list.get(1); + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return false; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.is(str); + } + } + + @Stellar(name = "DECODE", + description = "Decodes the passed string with the provided encoding, " + + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", + params = {"string - the string to decode", + "encoding - the encoding to use, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS", + "verify - (optional), true or false to determine if string should be verified as being " + + "encoded with the passed encoding" + }, + returns = "The decoded string on success\n" + + "The original string the string cannot be decoded\n" + + "null on usage error" + ) + public static class Decode extends BaseStellarFunction { + + @Override + public Object apply(List<Object> list) { + if (list.size() != 2 && list.size() != 3) { + throw new IllegalStateException( + "DECODE expects two or three args: [string, encoding] or " + + "[string, encoding, verify] where encoding is one from " + + "the supported list"); + } + Boolean verify = false; + String str = (String) list.get(0); + String encoding = (String) list.get(1); + + if (list.size() == 3) { + verify = (Boolean)list.get(2); + } + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return null; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.decode(str, verify); + } + } + + @Stellar(name = "ENCODE", + description = "Encodes the passed string with the provided encoding, " + + " must be one of the encodings returned from LIST_SUPPORTED_ENCODINGS", + params = {"string - the string to encode", + "encoding - the encoding to use, must be one of encodings returned from " + + "LIST_SUPPORTED_ENCODINGS" + }, + returns = "The encoded string or null on error" + ) + public static class Encode extends BaseStellarFunction { + + @Override + public Object apply(List<Object> list) { + if (list.size() != 2 && list.size() != 3) { + throw new IllegalStateException( + "ENCODE expects two or three args: [string, encoding] where encoding is one from " + + "the supported list"); + } + String str = (String) list.get(0); + String encoding = (String) list.get(1); + + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(encoding)) { + return null; + } + + Encodings enc = null; + try { + enc = Encodings.valueOf(encoding.toUpperCase()); + } catch (IllegalArgumentException iae) { + throw new IllegalStateException(String.format("Encoding %s not supported", encoding), iae); + } + return enc.encode(str); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/encoding/EncodingsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/encoding/EncodingsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/encoding/EncodingsTest.java new file mode 100644 index 0000000..59896cc --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/encoding/EncodingsTest.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.encoding; + +import org.junit.Assert; +import org.junit.Test; + +public class EncodingsTest { + + public static final String STRING_FIXTURE = "Hello World"; + public static final String STRING_FIXTURE_PLUS_NULL = "Hello World\0"; + public static final String BASE32HEX_FIXTURE = "91IMOR3F41BMUSJCCG======"; + public static final String BASE32_FIXTURE = "JBSWY3DPEBLW64TMMQ======"; + public static final String BASE64_FIXTURE = "SGVsbG8gV29ybGQ="; + public static final String BINARY_FIXTURE = "0110010001101100011100100110111101010111001000000110111101101100011011000110010101001000"; + public static final String HEX_FIXTURE = "48656c6c6f20576f726c64"; + + @Test + public void is() throws Exception { + + // base32 + Assert.assertTrue(Encodings.BASE32.is(BASE32_FIXTURE)); + Assert.assertFalse(Encodings.BASE32.is(STRING_FIXTURE)); + + // base32 hex + Assert.assertTrue(Encodings.BASE32HEX.is(BASE32HEX_FIXTURE)); + Assert.assertFalse(Encodings.BASE32HEX.is(STRING_FIXTURE)); + + // base 64 + Assert.assertTrue(Encodings.BASE64.is(BASE64_FIXTURE)); + Assert.assertFalse(Encodings.BASE64.is(STRING_FIXTURE + "\0")); + + // binary + Assert.assertTrue(Encodings.BINARY.is(BINARY_FIXTURE)); + Assert.assertFalse(Encodings.BINARY.is(STRING_FIXTURE)); + + // hex + Assert.assertTrue(Encodings.HEX.is(HEX_FIXTURE)); + Assert.assertFalse(Encodings.HEX.is("AAA")); + } + + @Test + public void decode() throws Exception { + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32.decode(BASE32_FIXTURE)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32HEX.decode(BASE32HEX_FIXTURE)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE64.decode(BASE64_FIXTURE)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BINARY.decode(BINARY_FIXTURE)); + Assert.assertEquals(STRING_FIXTURE,Encodings.HEX.decode(HEX_FIXTURE)); + + // these codecs will just decode away... and return garbage without verification + Assert.assertNotEquals(STRING_FIXTURE,Encodings.BASE32.decode(STRING_FIXTURE)); + Assert.assertNotEquals(STRING_FIXTURE,Encodings.BASE32HEX.decode(STRING_FIXTURE)); + Assert.assertNotEquals(STRING_FIXTURE,Encodings.BASE64.decode(STRING_FIXTURE)); + + // these codecs will fail to decode and return the original string without + // verification + Assert.assertEquals(STRING_FIXTURE,Encodings.BINARY.decode(STRING_FIXTURE)); + Assert.assertEquals(STRING_FIXTURE,Encodings.HEX.decode(STRING_FIXTURE)); + } + + @Test + public void decodeWithVerify() throws Exception { + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32.decode(BASE32_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32HEX.decode(BASE32HEX_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE64.decode(BASE64_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BINARY.decode(BINARY_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.HEX.decode(HEX_FIXTURE,true)); + + // with verification, we will get back the original string + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32.decode(STRING_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BASE32HEX.decode(STRING_FIXTURE,true)); + // if the string IS coincidentally compatable with base64, then it will decode away + Assert.assertNotEquals(STRING_FIXTURE,Encodings.BASE64.decode(STRING_FIXTURE,true)); + // if the string would fail... then we get the original + Assert.assertEquals(STRING_FIXTURE + "\0",Encodings.BASE64.decode(STRING_FIXTURE + "\0",true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.BINARY.decode(STRING_FIXTURE,true)); + Assert.assertEquals(STRING_FIXTURE,Encodings.HEX.decode(STRING_FIXTURE,true)); + } + + @Test + public void testEncode() throws Exception{ + Assert.assertEquals(BASE32_FIXTURE,Encodings.BASE32.encode(STRING_FIXTURE)); + Assert.assertEquals(BASE32HEX_FIXTURE,Encodings.BASE32HEX.encode(STRING_FIXTURE)); + Assert.assertEquals(BASE64_FIXTURE,Encodings.BASE64.encode(STRING_FIXTURE)); + Assert.assertEquals(BINARY_FIXTURE,Encodings.BINARY.encode(STRING_FIXTURE)); + Assert.assertEquals(HEX_FIXTURE,Encodings.HEX.encode(STRING_FIXTURE)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index 871055a..06edb12 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -104,8 +104,6 @@ public class BasicStellarTest { Stellar annotation = clazz.getAnnotation(Stellar.class); Assert.assertFalse("Must specify a name for " + clazz.getName(),StringUtils.isEmpty(annotation.name())); Assert.assertFalse("Must specify a description annotation for " + clazz.getName(),StringUtils.isEmpty(annotation.description())); - Assert.assertTrue("Must specify a non-empty params for " + clazz.getName(), annotation.params().length > 0); - Assert.assertTrue("Must specify a non-empty params for " + clazz.getName(), StringUtils.isNoneEmpty(annotation.params())); Assert.assertFalse("Must specify a returns annotation for " + clazz.getName(), StringUtils.isEmpty(annotation.returns())); } } http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/EncodingFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/EncodingFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/EncodingFunctionsTest.java new file mode 100644 index 0000000..ca544aa --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/EncodingFunctionsTest.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import static org.apache.metron.stellar.common.encoding.EncodingsTest.BASE32HEX_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.BASE32_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.BASE64_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.BINARY_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.HEX_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.STRING_FIXTURE; +import static org.apache.metron.stellar.common.encoding.EncodingsTest.STRING_FIXTURE_PLUS_NULL; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections4.ListUtils; +import org.apache.metron.stellar.common.encoding.Encodings; +import org.junit.Assert; +import org.junit.Test; + +public class EncodingFunctionsTest { + + private static final Map<String, Object> variableMap = new HashMap<String, Object>() {{ + put("BASE32HEX_FIXTURE","91IMOR3F41BMUSJCCG======"); + put("BASE32_FIXTURE" , "JBSWY3DPEBLW64TMMQ======\r\n"); + put("BASE64_FIXTURE" , "SGVsbG8gV29ybGQ="); + put("BINARY_FIXTURE" , "0110010001101100011100100110111101010111001000000110111101101100011011000110010101001000"); + put("HEX_FIXTURE" , "48656c6c6f20576f726c64"); + put("STRING_FIXTURE", STRING_FIXTURE); + put("STRING_FIXTURE_PLUS_NULL", STRING_FIXTURE_PLUS_NULL); + + }}; + + @Test + public void testSupportedEncodingsList() throws Exception{ + Object ret = run("GET_SUPPORTED_ENCODINGS()", new HashMap()); + Assert.assertTrue(ret instanceof List ); + List<String> list = (List<String>)ret; + List<String> expected = new ArrayList<>(Arrays.asList("BASE32","BASE32HEX","BASE64","BINARY","HEX")); + Assert.assertTrue(ListUtils.isEqualList(expected,list)); + } + + @Test + public void testEncodingIs() throws Exception{ + Assert.assertTrue(runPredicate("IS_ENCODING(BASE32_FIXTURE,'BASE32')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IS_ENCODING(STRING_FIXTURE,'BASE32')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IS_ENCODING(BASE32HEX_FIXTURE,'BASE32HEX')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IS_ENCODING(STRING_FIXTURE,'BASE32HEX')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IS_ENCODING(BASE64_FIXTURE,'BASE64')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IS_ENCODING(STRING_FIXTURE_PLUS_NULL,'BASE64')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IS_ENCODING(BINARY_FIXTURE,'BINARY')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IS_ENCODING(STRING_FIXTURE,'BINARY')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IS_ENCODING(HEX_FIXTURE,'HEX')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IS_ENCODING(STRING_FIXTURE,'HEX')", v -> variableMap.get(v))); + } + + @Test + public void testDecode() throws Exception { + Assert.assertEquals(STRING_FIXTURE,run("DECODE(BASE32_FIXTURE,'BASE32')",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BASE32HEX_FIXTURE,'BASE32HEX')",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BASE64_FIXTURE,'BASE64')",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BINARY_FIXTURE,'BINARY')",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(HEX_FIXTURE,'HEX')",variableMap)); + + // these codecs will just decode away... and return garbage without verification + Assert.assertNotEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE32')",variableMap)); + Assert.assertNotEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE32HEX')",variableMap)); + Assert.assertNotEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE64')",variableMap)); + + // these codecs will fail to decode and return the original string without + // verification + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BINARY')",variableMap)); + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE, 'HEX')", variableMap)); + } + + @Test + public void testDecodeWithVerify() throws Exception { + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BASE32_FIXTURE,'BASE32',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BASE32HEX_FIXTURE,'BASE32HEX',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BASE64_FIXTURE,'BASE64',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(BINARY_FIXTURE,'BINARY',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE, run("DECODE(HEX_FIXTURE,'HEX',true)",variableMap)); + + + + // with verification, we will get back the original string + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE32',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE32HEX',true)",variableMap)); + // if the string IS coincidentally compatable with base64, then it will decode away + Assert.assertNotEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BASE64',true)",variableMap)); + // if the string would fail... then we get the original + Assert.assertEquals(STRING_FIXTURE_PLUS_NULL,run("DECODE(STRING_FIXTURE_PLUS_NULL,'BASE64',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'BINARY',true)",variableMap)); + Assert.assertEquals(STRING_FIXTURE,run("DECODE(STRING_FIXTURE,'HEX',true)",variableMap)); + } + + @Test + public void testEncode() throws Exception { + Assert.assertEquals(BASE32_FIXTURE,run("ENCODE(STRING_FIXTURE,'BASE32')",variableMap)); + Assert.assertEquals(BASE32HEX_FIXTURE, run("ENCODE(STRING_FIXTURE,'BASE32HEX')",variableMap)); + Assert.assertEquals(BASE64_FIXTURE, run("ENCODE(STRING_FIXTURE,'BASE64')",variableMap)); + Assert.assertEquals(BINARY_FIXTURE, run("ENCODE(STRING_FIXTURE,'BINARY')",variableMap)); + Assert.assertEquals(HEX_FIXTURE, run("ENCODE(STRING_FIXTURE,'HEX')",variableMap)); + } +} \ No newline at end of file
