[43/51] [abbrv] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
new file mode 100644
index 000..5f05b24
--- /dev/null
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.error.MetronError;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.filters.Filters;
+import org.apache.metron.parsers.filters.StellarFilter;
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.topology.ParserComponent;
+import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ParserRunnerImpl.class, ReflectionUtils.class, Filters.class})
+public class ParserRunnerImplTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ /**
+ {
+ "fieldValidations" : [
+ {
+ "input" : [ "ip_src_addr", "ip_dst_addr"],
+ "validation" : "IP"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String globalConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro",
+ "parserConfig": {
+ "field": "value"
+ },
+ "fieldTransformations" : [
+ {
+ "input" : "field1",
+ "transformation" : "REMOVE"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String broConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
+ "sensorTopic":"snort",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ private String snortConfigString;
+
+ private ParserConfigurations parserConfigurations;
+ private MessageParser broParser;
+ private MessageParser snortParser;
+ private MessageFilter stellarFilter;
+ private ParserRunnerImpl parserRunner;
+
+
+ @Before
+ public void setup() throws IOException {
+parserConfigurations = new ParserConfigurations();
+SensorParserConfig broConfig =
SensorParserConfig.fromBytes(broConfigString.getBytes());
+SensorParserConfig snortConfig =
SensorParserConfig.fromBytes(snortConfigString.getBytes());
+
[48/51] [abbrv] metron git commit: METRON-1833: Management UI incorrectly displaying sensor topology latency units as seconds instead of millis (mmiklavc via mmiklavc) closes apache/metron#1241
METRON-1833: Management UI incorrectly displaying sensor topology latency units
as seconds instead of millis (mmiklavc via mmiklavc) closes apache/metron#1241
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c0602c7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c0602c7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c0602c7
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0c0602c75d9660717920a547bcb9e76d58b6571e
Parents: d44a392
Author: mmiklavc
Authored: Mon Oct 22 12:54:26 2018 -0600
Committer: Michael Miklavcic
Committed: Mon Oct 22 12:54:26 2018 -0600
--
.../sensor-parser-config-readonly.component.ts | 2 +-
.../sensor-parser-list/sensor-parser-list.component.spec.ts| 2 +-
.../app/sensors/sensor-parser-list/sensor-parser-list.component.ts | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
index 5db6d45..7d41003 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
@@ -156,7 +156,7 @@ export class SensorParserConfigReadonlyComponent implements
OnInit {
getTopologyStatus(key: string): string {
if (key === 'latency') {
- return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency +
's') : '-';
+ return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency +
'ms') : '-';
} else if (key === 'throughput') {
return this.topologyStatus.throughput >= 0 ?
((Math.round(this.topologyStatus.throughput * 100) / 100) + 'kb/s') : '-';
} else if (key === 'emitted') {
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
index 205d885..fb2a175 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
@@ -720,7 +720,7 @@ describe('Component: SensorParserList', () => {
component.updateSensorStatus();
expect(component.sensors[0]['status']).toEqual('Running');
-expect(component.sensors[0]['latency']).toEqual('10s');
+expect(component.sensors[0]['latency']).toEqual('10ms');
expect(component.sensors[0]['throughput']).toEqual('23kb/s');
component.sensorsStatus[0].status = 'KILLED';
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
index 1129914..2694ab4 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
@@ -163,7 +163,7 @@ export class SensorParserListComponent implements OnInit {
sensor['status'] = 'Stopped';
}
-sensor['latency'] = status && status.status === 'ACTIVE' ?
(status.latency + 's') : '-';
+sensor['latency'] = status && status.status === 'ACTIVE' ?
(status.latency + 'ms') : '-';
sensor['throughput'] = status && status.status === 'ACTIVE' ?
(Math.round(status.throughput * 100) / 100) + 'kb/s' : '-';
}
}
[46/51] [abbrv] metron git commit: METRON-1831 Project Version Substitution Not Working (nickwallen) closes apache/metron#1243
METRON-1831 Project Version Substitution Not Working (nickwallen) closes
apache/metron#1243
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2531c3e1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2531c3e1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2531c3e1
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 2531c3e19e6c2887a2242da830d7178e6973b8ab
Parents: bf32fef
Author: nickwallen
Authored: Sat Oct 20 11:48:38 2018 -0400
Committer: nickallen
Committed: Sat Oct 20 11:48:38 2018 -0400
--
metron-platform/metron-common/src/main/scripts/stellar | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2531c3e1/metron-platform/metron-common/src/main/scripts/stellar
--
diff --git a/metron-platform/metron-common/src/main/scripts/stellar
b/metron-platform/metron-common/src/main/scripts/stellar
index 34097f2..690a9f0 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -33,8 +33,9 @@ if [ -f "$METRON_SYSCONFIG" ]; then
source $METRON_SYSCONFIG
fi
-export METRON_VERSION="${METRON_VERSION:-${project.version}}"
-export METRON_HOME="${METRON_HOME:-/usr/metron/$METRON_VERSION}"
+# treat unset vars as an error; METRON_HOME
+set -u
+
export HBASE_CONFIGS=$(hbase classpath)
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
[38/51] [abbrv] metron git commit: METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216
METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b671005 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b671005 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b671005 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 9b6710053894d8a39880cb8157a0e603ed542cb7 Parents: f153375 Author: justinleet Authored: Thu Oct 11 08:41:23 2018 -0400 Committer: leet Committed: Thu Oct 11 08:41:23 2018 -0400 -- site/develop/bylaws.md | 281 site/develop/coding.md | 34 -- site/develop/index.md | 61 -- 3 files changed, 376 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/bylaws.md -- diff --git a/site/develop/bylaws.md b/site/develop/bylaws.md deleted file mode 100644 index a8fc5fb..000 --- a/site/develop/bylaws.md +++ /dev/null @@ -1,281 +0,0 @@ -layout: page -title: Apache Metron Bylaws - -## Introduction - -This document defines the bylaws under which the Apache Metron project -operates. It defines the roles and responsibilities of the project, -who may vote, how voting works, how conflicts are resolved, etc. - -Metron is a project of the Apache Software Foundation (ASF) and the foundation -holds the trademark on the name "Metron" and copyright on the combined -code base. The [Apache Foundation -FAQ](http://www.apache.org/foundation/faq.html) and -[How-It-Works](http://www.apache.org/foundation/how-it-works.html) -explain the operation and background of the foundation. - -Apache has a [code of -conduct](http://www.apache.org/foundation/policies/conduct.html) that -it expects its members to follow. In particular: - -* Be **open and welcoming**. It is important that we grow and - encourage the community of users and developers for our project. - -* Be **collaborative**. Working together on the open mailing lists and - bug database to make decisions helps the project grow. - -* Be **respectful** of others. Everyone is volunteering their time and - efforts to work on this project. Please be respectful of everyone - and their views. - -Metron is typical of Apache projects in that it operates under a set of -principles, known collectively as the "Apache Way". If you are new to -Apache development, please refer to -[this](http://www.apache.org/foundation/how-it-works.html) for more -information on how Apache projects operate. - -## Roles and Responsibilities - -Apache projects define a set of roles with associated rights and -responsibilities. These roles govern what tasks an individual may -perform within the project. The roles are defined in the following -sections. - -### Users - -The most important participants in the project are people who use our -software. The majority of our developers start out as users and guide -their development efforts from the user's perspective. Users -contribute to the Apache projects by providing feedback to developers -in the form of bug reports and feature suggestions. As well, users -participate in the Apache community by helping other users on mailing -lists and user support forums. - -### Contributors - -Contributors include all of the volunteers who donate time, code, -documentation, -or resources to the Metron Project. A contributor that makes sustained, -welcome contributions to the project may be invited to become a -committer, though the exact timing of such invitations depends on many -factors. - -### Committers - -The project's committers are responsible for the project's technical -management. Committers have the right to commit to the project's git -repository. Committers may cast binding votes on any technical -discussion. - -Committer access is by invitation only and must be approved by -consensus approval of the active Project Management Committee (PMC) -members. - -If a committer wishes to leave the project or does not contribute to -the project in any form for six months, the PMC may make them emeritus. -Emeritus committers lose their ability to commit code or cast binding -votes. An emeritus committer may -request reinstatement of commit access from the PMC. Such -reinstatement is subject to consensus approval of active PMC members. - -All Apache committers are required to have a signed [Individual -Contributor License -Agreement](https://www.apache.org/licenses/icla.txt) (ICLA) on file -with the Apache Software Foundation. There is a [Committer -FAQ](http://www.apache.org/dev/committers.html) which provides more -details on the requirements for Committers. - -A committer who makes a -sustained contribution to the project may be invited to become a -member
[37/51] [abbrv] metron git commit: METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188
METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f153375f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f153375f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f153375f Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: f153375fad65e75a10ce490b4a5f29f7e82f70dd Parents: 3d923cd Author: justinleet Authored: Wed Oct 10 15:41:22 2018 -0400 Committer: leet Committed: Wed Oct 10 15:41:22 2018 -0400 -- dev-utilities/release-utils/README.md | 129 +++ dev-utilities/release-utils/metron-rc-check | 6 +- .../release-utils/prepare-release-candidate | 337 +++ 3 files changed, 469 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/README.md -- diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md new file mode 100644 index 000..d1fd816 --- /dev/null +++ b/dev-utilities/release-utils/README.md @@ -0,0 +1,129 @@ + + +# Release Tools + +This project contains tools to assist Apache Metron project committers. + +## Prepare Release Candidate + +This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary. Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg) + +When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done. + +In the following example, enter the appropriate information + +1. Execute the script. + +The first time the script is run, you will be prompted for additional information including your Apache username and Apache email. These values are persisted in `~/.metron-prepare-release-candidate`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them. + +``` +$ ./prepare-release-candidate + your apache userid []: leet + your apache email [l...@apache.org]: +``` + +1. Select a repository we're creating an RC for. + +``` +[1] metron +[2] metron-bro-plugin-kafka + which repo? [1]: 1 +``` + +1. Enter the current version number. This will be the base for the CHANGES file + +``` + current version: 0.6.0 +``` + +1. Enter the version being built. + +``` + version being built: 0.6.1 +``` + +1. Enter the current RC number + +``` + release candidate number: 1 +``` + +1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch. + +``` + base revision branch or hash for release candidate [master]: +``` + +1. Enter the signing key id. + +``` + signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): +``` + +1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise. + +``` + do a live run (push to remote repositories?) [y/n] +``` + +1. Wait for all repos to be checked out to complete. There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch. + +``` + Checking out repo: https://dist.apache.org/repos/dist/dev/metron + Checking out repo: dev + Checking out repo: https://dist.apache.org/repos/dist/release/metron + Checking out repo: release + Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git + Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'... + remote: Counting objects: 46146, done. + remote: Compressing objects: 100% (15568/15568), done. + remote: Total 46146 (delta 21513), reused 43696 (delta 19489) + Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done. + Resolving deltas: 100% (21513/21513), done. + Creating branch: Metron_0.6.1 + Using git rev: master + Already on 'master' + Your branch is up to date with 'origin/master'. + Switched to a new branch 'Metron_0.6.1' + This is a practice run. Not running + Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review. + Already on 'Metron_0.6.1' + Creating the RC tarball for tag apache-metron-0.6.1-rc1 + Creating the SHA
[19/51] [abbrv] metron git commit: METRON-1794 Include User Details When Escalating Alerts (nickwallen) closes apache/metron#1212
METRON-1794 Include User Details When Escalating Alerts (nickwallen) closes
apache/metron#1212
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c0fb2625
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c0fb2625
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c0fb2625
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: c0fb26258f9a78d375a50dd5d746404130dfe06b
Parents: 9c9e295
Author: nickwallen
Authored: Mon Oct 1 09:49:17 2018 -0400
Committer: nickallen
Committed: Mon Oct 1 09:49:17 2018 -0400
--
.../apache/metron/rest/MetronRestConstants.java | 3 +
.../rest/service/impl/AlertsUIServiceImpl.java | 53
.../service/impl/AlertsUIServiceImplTest.java | 63
.../src/test/resources/log4j.properties | 5 +-
4 files changed, 98 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index e3bf698..94e8e35 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -53,6 +53,9 @@ public class MetronRestConstants {
public static final String KAFKA_BROKER_URL_SPRING_PROPERTY =
"kafka.broker.url";
public static final String KAFKA_TOPICS_ESCALATION_PROPERTY =
"kafka.topics.escalation";
+ public static final String METRON_ESCALATION_USER_FIELD =
"metron_escalation_user";
+ public static final String METRON_ESCALATION_TIMESTAMP_FIELD =
"metron_escalation_timestamp";
+
public static final String KERBEROS_ENABLED_SPRING_PROPERTY =
"kerberos.enabled";
public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY =
"kerberos.principal";
public static final String KERBEROS_KEYTAB_SPRING_PROPERTY =
"kerberos.keytab";
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
index 7d0a8f8..bf035e2 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
@@ -19,26 +19,32 @@ package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.metron.common.system.Clock;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.AlertsUIUserSettings;
-import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.security.SecurityUtils;
import org.apache.metron.rest.service.AlertsUIService;
import org.apache.metron.rest.service.KafkaService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static
org.apache.metron.rest.MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY;
+import static
org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD;
+import static
org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_USER_FIELD;
+
/**
* The default service layer implementation of {@link AlertsUIService}.
*
@@ -47,6 +53,7 @@ import org.springframework.stereotype.Service;
@Service
public class AlertsUIServiceImpl implements AlertsUIService {
+ static final Logger LOG =
[42/51] [abbrv] metron git commit: METRON-1820 Update to new Simple-Syslog-5424 version to support error handling (ottobackwards) closes apache/metron#1234
METRON-1820 Update to new Simple-Syslog-5424 version to support error handling
(ottobackwards) closes apache/metron#1234
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08f3de0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08f3de0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08f3de0f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08f3de0fe31fefa828952cbe76456580a4697630
Parents: 6214150
Author: ottobackwards
Authored: Mon Oct 15 16:36:36 2018 -0400
Committer: otto
Committed: Mon Oct 15 16:36:36 2018 -0400
--
dependencies_with_url.csv | 2 +-
.../impl/SensorParserConfigServiceImpl.java | 65 +-
.../org/apache/metron/parsers/GrokParser.java | 18 +---
.../apache/metron/parsers/bolt/ParserBolt.java | 37 +++-
.../parsers/interfaces/MessageParser.java | 18
.../interfaces/MultilineMessageParser.java | 51 +++
.../metron/parsers/syslog/Syslog5424Parser.java | 44 --
.../apache/metron/parsers/GrokParserTest.java | 8 +-
.../metron/parsers/MultiLineGrokParserTest.java | 5 +-
.../metron/parsers/bolt/ParserBoltTest.java | 91 +++-
.../parsers/syslog/Syslog5424ParserTest.java| 55 +---
.../websphere/GrokWebSphereParserTest.java | 56
pom.xml | 2 +-
13 files changed, 335 insertions(+), 117 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index fb6c03c..2e1eedd 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -484,4 +484,4 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
+com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 85b84b8..4cd272e 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
@@ -33,6 +34,8 @@ import
org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
@@ -138,13 +141,53 @@ public class SensorParserConfigServiceImpl implements
SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a
parserClassName");
} else {
- MessageParser parser;
+ MultilineMessageParser parser;
+ Object parserObject;
try {
-parser = (MessageParser)
Class.forName(sensorParserConfig.getParserClassName())
+parserObject = Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
+
+ if (!(parserObject instanceof MultilineMessageParser)) {
+parser = new MultilineMessageParser() {
+
+ @Override
+
[25/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
--
diff --git
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
index 4d48075..096baf1 100644
---
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
+++
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
@@ -24,16 +24,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
@@ -80,15 +81,16 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
* @param retrieveLatestDao DAO to retrieve the item to be patched
* @param request The patch request.
* @param timestamp Optionally a timestamp to set. If not specified then
current time is used.
+ * @return The patched document
* @throws OriginalNotFoundException If no original document is found to
patch.
* @throws IOException If an error occurs performing the patch.
*/
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest
request,
Optional timestamp)
throws OriginalNotFoundException, IOException {
if (isPatchAllowed(request)) {
- updateDao.patch(retrieveLatestDao, request, timestamp);
+ return updateDao.patch(retrieveLatestDao, request, timestamp);
} else {
throw new IllegalArgumentException(
"Meta alert patches are not allowed for /alert or /status paths. "
@@ -97,7 +99,7 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
}
@Override
- public void batchUpdate(Map> updates) {
+ public Map> batchUpdate(Map> updates) {
throw new UnsupportedOperationException("Meta alerts do not allow for bulk
updates");
}
@@ -170,21 +172,65 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
return updates;
}
+ /**
+ * Adds alerts to a metaalert, based on a list of GetRequests provided for
retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to be given new children.
+ * @param alertRequests GetRequests for the appropriate alerts to add.
+ * @return The updated metaalert with alerts added.
+ */
+ @Override
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List
alertRequests)
+ throws IOException {
+Document metaAlert = retrieveLatestDao
+.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
+if (metaAlert == null) {
+ throw new IOException(String.format("Unable to add alerts to meta alert.
Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
+}
+if (MetaAlertStatus.ACTIVE.getStatusString()
+
.equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
+ Iterable alerts =
retrieveLatestDao.getAllLatest(alertRequests);
+ Set missingAlerts = getMissingAlerts(alertRequests, alerts);
+ if (!missingAlerts.isEmpty()) {
+throw new IOException(String.format("Unable to add alerts to meta
alert. Alert with guid %s cannot be found.",
+missingAlerts.iterator().next()));
+ }
+ Map> updates =
buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
+ update(updates);
+ return metaAlert;
+} else {
+ throw new IllegalStateException("Adding alerts to an INACTIVE meta alert
is not allowed");
+}
+ }
+
+ /**
+ * Removes alerts from a metaalert, based on a list of GetRequests provided
for retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to remove children from.
+ * @param alertRequests A list of GetReqests that will provide the alerts to
remove
+ * @return The updated metaalert with alerts removed.
+ * @throws IllegalStateException If the metaalert is inactive.
+ */
[33/51] [abbrv] metron git commit: METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes apache/metron#1231
METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes
apache/metron#1231
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/747220f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/747220f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/747220f0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 747220f000188557bfa860b48f6eb4bc49758d4a
Parents: 3467ffd
Author: merrimanr
Authored: Tue Oct 9 15:33:39 2018 -0500
Committer: rmerriman
Committed: Tue Oct 9 15:33:39 2018 -0500
--
.../CURRENT/package/files/bro_index.template| 3 +
.../CURRENT/package/files/snort_index.template | 3 +
.../CURRENT/package/files/yaf_index.template| 3 +
metron-platform/metron-elasticsearch/README.md | 20 +-
.../ElasticsearchSearchIntegrationTest.java | 211 ++-
.../indexing/dao/SearchIntegrationTest.java | 34 +--
.../src/main/config/schema/bro/schema.xml | 1 +
.../src/main/config/schema/snort/schema.xml | 1 +
.../src/main/config/schema/yaf/schema.xml | 1 +
.../integration/SolrSearchIntegrationTest.java | 8 +-
10 files changed, 112 insertions(+), 173 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 17ad4d2..48e76c7 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -133,6 +133,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
/*
* Widely-used Bro fields (potentially renamed during Metron ingest)
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index f7c6e59..139dbc9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index f4093ba..4a3e5f9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/README.md
--
diff --git a/metron-platform/metron-elasticsearch/README.md
b/metron-platform/metron-elasticsearch/README.md
index d889e27..177412e 100644
---
[44/51] [abbrv] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr)
closes apache/metron#1213
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28542ad6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28542ad6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28542ad6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 28542ad64cf63f17b728b4b1c0e995a8973767f7
Parents: 08f3de0
Author: merrimanr
Authored: Thu Oct 18 13:59:52 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 13:59:52 2018 -0500
--
.../impl/SensorParserConfigServiceImpl.java | 51 +-
.../parsers/DefaultParserRunnerResults.java | 71 ++
.../org/apache/metron/parsers/GrokParser.java | 3 +-
.../org/apache/metron/parsers/ParserRunner.java | 60 ++
.../apache/metron/parsers/ParserRunnerImpl.java | 322 +++
.../metron/parsers/ParserRunnerResults.java | 33 +
.../apache/metron/parsers/bolt/ParserBolt.java | 381 +++-
.../parsers/filters/BroMessageFilter.java | 2 +-
.../metron/parsers/filters/StellarFilter.java | 2 +-
.../parsers/interfaces/MessageFilter.java | 2 +-
.../parsers/interfaces/MessageParser.java | 27 +-
.../interfaces/MultilineMessageParser.java | 51 --
.../metron/parsers/syslog/Syslog5424Parser.java | 4 +-
.../parsers/topology/ParserComponent.java | 56 ++
.../parsers/topology/ParserComponents.java | 67 --
.../parsers/topology/ParserTopologyBuilder.java | 39 +-
.../org/apache/metron/filters/FiltersTest.java | 4 +-
.../metron/parsers/MessageParserTest.java | 108 ++-
.../metron/parsers/ParserRunnerImplTest.java| 390 +
.../metron/parsers/bolt/ParserBoltTest.java | 859 ++-
.../parsers/integration/ParserDriver.java | 60 +-
21 files changed, 1481 insertions(+), deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 4cd272e..d0e4b3d 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -20,12 +20,10 @@ package org.apache.metron.rest.service.impl;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -35,18 +33,14 @@ import
org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.metron.rest.util.ParserIndex;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -141,53 +135,13 @@ public class SensorParserConfigServiceImpl implements
SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a
parserClassName");
} else {
- MultilineMessageParser parser;
- Object parserObject;
+ MessageParser parser;
try {
-parserObject = Class.forName(sensorParserConfig.getParserClassName())
+parser = (MessageParser)
Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new
[27/51] [abbrv] metron git commit: METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes apache/metron#1207
METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes
apache/metron#1207
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/feb9153a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/feb9153a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/feb9153a
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: feb9153a610e7cc384055c5d07b5c1dcf8b81923
Parents: de53306
Author: anandsubbu
Authored: Fri Oct 5 12:09:55 2018 +0530
Committer: anandsubbu
Committed: Fri Oct 5 12:09:55 2018 +0530
--
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 7 +
.../CURRENT/configuration/metron-pcap-env.xml | 188 +++
.../CURRENT/configuration/metron-rest-env.xml | 47 -
.../common-services/METRON/CURRENT/metainfo.xml | 3 +-
.../package/scripts/params/params_linux.py | 33 +++-
.../package/scripts/params/status_params.py | 2 +-
.../CURRENT/package/scripts/pcap_master.py | 7 +
.../METRON/CURRENT/themes/metron_theme.json | 174 +++--
metron-interface/metron-rest/README.md | 4 +-
metron-platform/metron-pcap-backend/README.md | 3 +
.../src/main/assembly/assembly.xml | 1 +
.../src/main/config/pcap.properties | 9 +-
.../src/main/config/pcap.properties.j2 | 40
15 files changed, 446 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 2ec956a..f67522c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -105,7 +105,7 @@ configurations:
- metron-rest-env:
metron_spring_profiles_active: "dev"
- metron-parsers-env:
- parsers: "bro,snort"
+ parsers: "\"bro,snort,yaf\""
- elastic-site:
index_number_of_shards: 1
index_number_of_replicas: 0
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/.gitignore
--
diff --git a/metron-deployment/packaging/ambari/.gitignore
b/metron-deployment/packaging/ambari/.gitignore
index d2cc43c..242a4da 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -6,3 +6,4 @@ hdfs.properties.j2
enrichment.properties.j2
enrichment-splitjoin.properties.j2
enrichment-unified.properties.j2
+pcap.properties.j2
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
--
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 82386c1..2975963 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -136,6 +136,13 @@
false
+
+
${basedir}/../../../../metron-platform/metron-pcap-backend/src/main/config
+
+pcap.properties.j2
+
+false
+
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
new file mode 100644
index 000..31f9e8f
--- /dev/null
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
@@ -0,0 +1,188 @@
+
+
+
+
+
+pcap_topology_worker_childopts
+PCAP Topology JVM Options
+
[26/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de533063 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de533063 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de533063 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: de533063c26c8a2462dbfff33228a949c4a96c97 Parents: 7e222fa Author: merrimanr Authored: Thu Oct 4 17:16:18 2018 -0500 Committer: rmerriman Committed: Thu Oct 4 17:16:18 2018 -0500 -- metron-interface/metron-rest/README.md | 16 ++ .../rest/controller/MetaAlertController.java| 18 +- .../rest/controller/UpdateController.java | 31 ++- .../metron/rest/service/MetaAlertService.java | 12 +- .../metron/rest/service/UpdateService.java | 9 +- .../rest/service/impl/MetaAlertServiceImpl.java | 18 +- .../rest/service/impl/UpdateServiceImpl.java| 18 +- .../MetaAlertControllerIntegrationTest.java | 152 +++--- .../elasticsearch/dao/ElasticsearchDao.java | 32 +-- .../dao/ElasticsearchMetaAlertDao.java | 37 ++-- .../dao/ElasticsearchMetaAlertUpdateDao.java| 55 ++--- .../dao/ElasticsearchUpdateDao.java | 42 ++-- .../dao/ElasticsearchMetaAlertDaoTest.java | 27 ++- .../dao/ElasticsearchUpdateDaoTest.java | 52 + .../apache/metron/indexing/dao/HBaseDao.java| 31 +-- .../metron/indexing/dao/MultiIndexDao.java | 138 - .../dao/metaalert/MetaAlertCreateResponse.java | 40 .../dao/metaalert/MetaAlertUpdateDao.java | 20 +- .../AbstractLuceneMetaAlertUpdateDao.java | 81 +++- .../metron/indexing/dao/update/UpdateDao.java | 25 ++- .../InMemoryMetaAlertRetrieveLatestDao.java | 49 + .../metron/indexing/dao/HBaseDaoTest.java | 41 .../apache/metron/indexing/dao/InMemoryDao.java | 18 +- .../indexing/dao/InMemoryMetaAlertDao.java | 199 +-- .../dao/InMemoryMetaAlertUpdateDao.java | 91 + .../metron/indexing/dao/MultiIndexDaoTest.java | 96 + .../metron/indexing/dao/UpdateDaoTest.java | 74 +++ .../indexing/dao/UpdateIntegrationTest.java | 138 +++-- .../dao/metaalert/MetaAlertIntegrationTest.java | 119 --- .../AbstractLuceneMetaAlertUpdateDaoTest.java | 56 -- .../integration/HBaseDaoIntegrationTest.java| 7 +- .../org/apache/metron/solr/dao/SolrDao.java | 28 +-- .../metron/solr/dao/SolrMetaAlertDao.java | 49 +++-- .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 38 ++-- .../apache/metron/solr/dao/SolrUpdateDao.java | 36 ++-- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 21 +- .../metron/solr/dao/SolrUpdateDaoTest.java | 19 +- 37 files changed, 1206 insertions(+), 727 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/README.md -- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 2c216d1..2ce9522 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -336,6 +336,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)| | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)| | [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)| +| [ `POST /api/v1/update/add/comment`](#put-apiv1updateaddcomment)| +| [ `POST /api/v1/update/remove/comment`](#put-apiv1updateremovecomment)| | [ `GET /api/v1/user`](#get-apiv1user)| ### `POST /api/v1/alerts/ui/escalate` @@ -963,6 +965,20 @@ Request and Response objects are JSON formatted. The JSON schemas are available ``` * Returns: * 200 - Current user + +### `POST /api/v1/update/add/comment` + * Description: Add a comment to an alert + * Input: +* request - Comment add request + * Returns: +* 200 - Returns the complete alert document with comments added. + +### `POST /api/v1/update/remove/comment` + * Description: Remove a comment from an alert + * Input: +* request - Comment remove request + * Returns: +* 200 - Returns the complete alert document with comments removed. ### `GET /api/v1/user` * Description: Retrieves the current user http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java -- diff --git
[31/51] [abbrv] metron git commit: METRON-1806: Upgrade Maven Shade Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224
METRON-1806: Upgrade Maven Shade Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e4823667 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e4823667 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e4823667 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: e482366726b07173fbc9ff9b084b851596e13005 Parents: 5bfc08c Author: mmiklavc Authored: Tue Oct 9 09:22:22 2018 -0600 Committer: Michael Miklavcic Committed: Tue Oct 9 09:22:22 2018 -0600 -- .gitignore | 1 + pom.xml| 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/.gitignore -- diff --git a/.gitignore b/.gitignore index a0b9691..df3f030 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ target/ *.tmp *.bak *.class +*.pyc tmp/** tmp/**/* http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/pom.xml -- diff --git a/pom.xml b/pom.xml index a98de72..1e6adb0 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 6.6.2 1.10.19 1.7.0 -2.4.3 +3.2.0 2.7.4 2.0.14 3.0.2
[21/51] [abbrv] metron git commit: METRON-1798 Add mpack support for parser aggregation (anandsubbu) closes apache/metron#1215
METRON-1798 Add mpack support for parser aggregation (anandsubbu) closes
apache/metron#1215
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ebdaf5f9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ebdaf5f9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ebdaf5f9
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ebdaf5f905829d8ff59814bb3555666217f85abb
Parents: ff1f9cf
Author: anandsubbu
Authored: Wed Oct 3 18:57:48 2018 +0530
Committer: anandsubbu
Committed: Wed Oct 3 18:57:48 2018 +0530
--
.../configuration/metron-parsers-env.xml| 2 +-
.../CURRENT/package/scripts/parser_commands.py | 49 ++--
.../metron-parsers/ParserChaining.md| 18 +++
3 files changed, 63 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
index a9a498b..03a2594 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
@@ -21,7 +21,7 @@
parsers
bro,snort,yaf
-Metron parsers to deploy
+Metron parsers to deploy. You can also specify an
aggregated parser list by grouping them with double quotes. For example:
"parserA,parserB",parserC,parserD
Metron Parsers
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index 274306a..18780d9 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -20,6 +20,7 @@ limitations under the License.
import os
import re
+import shlex
import subprocess
import time
@@ -49,7 +50,45 @@ class ParserCommands:
# get list of parsers
def __get_parsers(self, params):
-return params.parsers.replace(' ', '').split(',')
+"""
+Combines the list of parser topics and sends a unique list to be used
for
+Kafka topic creation and the like.
+:param params:
+:return: List containing the names of unique parsers
+"""
+parserBatches = list(self.__get_aggr_parsers(params))
+parsers = ','.join(s.translate(None, '"') for s in parserBatches)
+# Get only the unique list of parser names
+parsers = list(set(parsers.split(',')))
+return parsers
+
+def __get_aggr_parsers(self, params):
+"""
+Fetches the list of aggregated (and regular) parsers and returns a
list.
+If the input list of parsers were "bro,snort,yaf", "bro,snort" and
yaf, for example,
+then this method will return ["bro,snort,yaf", "bro,snort", "yaf"]
+:param params:
+:return: List containing the names of parsers
+"""
+parserList = []
+parsers = shlex.shlex(params.parsers)
+for name in parsers:
+parserList.append(name.strip(','))
+return [s.translate(None, "'[]") for s in filter(None, parserList)]
+
+def get_parser_aggr_topology_names(self, params):
+"""
+Returns the names of regular and aggregated topologies as they would
run in storm
+An aggregated topology has the naming convention of 'parserA__parserB'.
+For example, a list of parsers like ["bro,snort", yaf] will be
returned as ["bro__snort", "yaf"]
+:param params:
+:return: List containing the names of parser topologies
+"""
+topologyName = []
+for parser in self.__get_aggr_parsers(params):
+
[45/51] [abbrv] metron git commit: METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bf32fef6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bf32fef6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bf32fef6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: bf32fef6a4a02c5b21ec1f6d97fbd83de78a996e
Parents: 28542ad
Author: merrimanr
Authored: Thu Oct 18 17:00:44 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 17:00:44 2018 -0500
--
metron-stellar/stellar-common/README.md | 9 +
.../stellar/dsl/functions/DateFunctions.java| 42
.../dsl/functions/DateFunctionsTest.java| 38 ++
3 files changed, 89 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 468f358..0f3bb6a 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -169,6 +169,7 @@ Where:
| [ `CHOP`](#chop)
|
| [ `CHOMP`](#chomp)
|
| [ `COUNT_MATCHES`](#count_matches)
|
+| [ `DATE_FORMAT`](#date_format)
| [ `DAY_OF_MONTH`](#day_of_month)
|
| [ `DAY_OF_WEEK`](#day_of_week)
|
| [ `DAY_OF_YEAR`](#day_of_year)
|
@@ -379,6 +380,14 @@ Where:
* substring/character - the substring or character to count, may be null.
* Returns: the number of non-overlapping occurrences, 0 if either
CharSequence is null.
+### `DATE_FORMAT`
+ * Description: Takes an epoch timestamp and converts it to a date format.
+ * Input:
+* format - DateTime format as a String.
+* timestampField - Optional epoch time in Long format. Defaults to now.
+* timezone - Optional timezone in String format.
+ * Returns: Formatted date.
+
### `DAY_OF_MONTH`
* Description: The numbered day within the month. The first day within the
month has a value of 1.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
index 212d6e9..17f5f8d 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
@@ -28,6 +28,7 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
@@ -109,6 +110,13 @@ public class DateFunctions {
return sdf.parse(date).getTime();
}
+ public static String getDateFormat(String format, Optional epochTime,
Optional timezone) {
+Long time = epochTime.orElseGet(System::currentTimeMillis);
+TimezonedFormat fmt = timezone.map(s -> new TimezonedFormat(format,
s)).orElseGet(() -> new TimezonedFormat(format));
+SimpleDateFormat sdf = formatCache.get(fmt).get();
+return sdf.format(new Date(time));
+ }
+
/**
* Stellar Function: TO_EPOCH_TIMESTAMP
@@ -144,6 +152,40 @@ public class DateFunctions {
}
}
+ @Stellar( name="DATE_FORMAT",
+ description = "Takes an epoch timestamp and converts it to a date
format.",
+ params = {"format - DateTime format as a String."
+ , "timestampField - Optional epoch time in Long format.
Defaults to now."
+ , "timezone - Optional timezone in String format."},
+ returns="Formatted date."
+ )
+ public static class DateFormat extends BaseStellarFunction {
+
+public Object apply(List objects) {
+ int size = objects.size();
+ Optional formatObj = Optional.ofNullable(objects.get(0));
+ Optional epochObj = Optional.empty();
+ Optional tzObj = Optional.empty();
+
[47/51] [abbrv] metron git commit: METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr) closes apache/metron#1239
METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr)
closes apache/metron#1239
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d44a3925
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d44a3925
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d44a3925
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: d44a392567e35022bfb35a787b55aff3447ac60e
Parents: 2531c3e
Author: merrimanr
Authored: Mon Oct 22 08:43:10 2018 -0500
Committer: rmerriman
Committed: Mon Oct 22 08:43:10 2018 -0500
--
.../bolt/BulkMessageWriterBoltTest.java | 2 +-
metron-platform/metron-writer/pom.xml | 6 ++
.../metron/writer/BulkWriterComponent.java | 33 ++-
.../writer/bolt/BulkMessageWriterBolt.java | 2 +-
.../metron/writer/BulkWriterComponentTest.java | 61 +++-
5 files changed, 76 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
--
diff --git
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 588fc58..083628c 100644
---
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -207,7 +207,7 @@ public class BulkMessageWriterBoltTest extends
BaseEnrichmentBoltTest {
}
UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
verify(outputCollector, times(5)).ack(tuple);
-verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
+verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/pom.xml
--
diff --git a/metron-platform/metron-writer/pom.xml
b/metron-platform/metron-writer/pom.xml
index e845516..6d08093 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -207,6 +207,12 @@
metron-common
${project.parent.version}
+
+org.apache.metron
+metron-test-utilities
+${project.parent.version}
+test
+
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
--
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 7678584..68585c5 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -40,6 +40,7 @@ import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,12 +119,18 @@ public class BulkWriterComponent {
public void error(String sensorType, Throwable e, Iterable tuples,
MessageGetStrategy messageGetStrategy) {
LOG.error(format("Failing %d tuple(s); sensorType=%s",
Iterables.size(tuples), sensorType), e);
-MetronError error = new MetronError()
-.withSensorType(Collections.singleton(sensorType))
-.withErrorType(Constants.ErrorType.INDEXING_ERROR)
-.withThrowable(e);
-tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t)));
-handleError(tuples, error);
+tuples.forEach(t -> {
+ MetronError error = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .addRawMessage(messageGetStrategy.get(t));
+ collector.emit(Constants.ERROR_STREAM, new
Values(error.getJSONObject()));
+ collector.ack(t);
+});
+//
[20/51] [abbrv] metron git commit: METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes
apache/metron#1175
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ff1f9cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ff1f9cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ff1f9cf5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ff1f9cf52b31ae866eb3de1cf8993af97af1790f
Parents: c0fb262
Author: ottobackwards
Authored: Mon Oct 1 10:23:38 2018 -0400
Committer: otto
Committed: Mon Oct 1 10:23:38 2018 -0400
--
dependencies_with_url.csv | 2 +-
.../docker/rpm-docker/SPECS/metron.spec | 3 +
.../data/syslog5424/parsed/Syslog5424Parsed | 3 +
.../sample/data/syslog5424/raw/Syslog5424Output | 3 +
metron-platform/metron-parsers/README.md| 5 +-
metron-platform/metron-parsers/pom.xml | 5 +
.../config/zookeeper/parsers/syslog5424.json| 7 +
.../metron/parsers/syslog/Syslog5424Parser.java | 102 +
.../Syslog5424ParserIntegrationTest.java| 37 +
.../parsers/syslog/Syslog5424ParserTest.java| 146 +++
pom.xml | 1 +
11 files changed, 311 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index ffd0fbc..53977f3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,4 +488,4 @@
com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-
+com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4bcef33..ed22a28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -158,6 +158,7 @@ This package installs the Metron Parser files
%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
@@ -590,6 +591,8 @@ chkconfig --del metron-alerts-ui
%changelog
* Thu Aug 30 2018 Apache Metron - 0.6.1
- Update compiled css file name for Alerts UI
+* Fri Aug 24 2018 Apache Metron - 0.6.1
+- Add syslog5424 parser
* Tue Aug 21 2018 Apache Metron - 0.6.1
- Add Profiler for REPL
* Tue Aug 14 2018 Apache Metron - 0.5.1
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
--
diff --git
a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
new file mode 100644
index 000..e330204
--- /dev/null
+++
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -0,0 +1,3 @@
+{"syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","syslog.header.hostName":"loggregator","original_string":"<14>1
2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA
- - Removing
instance","syslog.header.facility":"1","syslog.header.msgId":"-","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","timestamp":"2014-06-20T09:14:07+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structureddata.examples...@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other
[28/51] [abbrv] metron git commit: METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228
METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/891bf495 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/891bf495 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/891bf495 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 891bf4952ab8eceabcb44659e7498d62005b6aa2 Parents: feb9153 Author: justinleet Authored: Fri Oct 5 16:14:20 2018 -0400 Committer: leet Committed: Fri Oct 5 16:14:20 2018 -0400 -- metron-deployment/.gitignore | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/891bf495/metron-deployment/.gitignore -- diff --git a/metron-deployment/.gitignore b/metron-deployment/.gitignore index 6038b6b..f3d4a2a 100644 --- a/metron-deployment/.gitignore +++ b/metron-deployment/.gitignore @@ -1,3 +1,4 @@ keys/ .vagrant *.retry +*.pyc
[18/51] [abbrv] metron git commit: METRON-1782 Add Kafka Partition and Offset to Profiler Debug Logs (nickwallen) closes apache/metron#1202
METRON-1782 Add Kafka Partition and Offset to Profiler Debug Logs (nickwallen)
closes apache/metron#1202
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c9e2954
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c9e2954
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c9e2954
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9c9e29547837e648723920329c3b7fea6211f0db
Parents: 1723a0e
Author: nickwallen
Authored: Mon Oct 1 09:07:28 2018 -0400
Committer: nickallen
Committed: Mon Oct 1 09:07:28 2018 -0400
--
.../src/main/flux/profiler/remote.yaml | 11 +--
.../metron/profiler/storm/ProfileSplitterBolt.java | 13 -
.../metron/profiler/storm/ProfileSplitterBoltTest.java | 3 ++-
.../storm/kafka/flux/SimpleStormKafkaBuilder.java | 4 +++-
4 files changed, 26 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
--
diff --git
a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
index da71b27..e16a782 100644
--- a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
@@ -77,8 +77,15 @@ components:
className: "java.util.ArrayList"
configMethods:
- name: "add"
-args:
-- "value"
+args: ["value"]
+- name: "add"
+args: ["topic"]
+- name: "add"
+args: ["partition"]
+- name: "add"
+args: ["offset"]
+- name: "add"
+args: ["timestamp"]
- id: "kafkaConfig"
className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder"
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
--
diff --git
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
index 81179b6..ef58ad9 100644
---
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
+++
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
@@ -43,6 +43,12 @@ import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.OFFSET;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.PARTITION;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TIMESTAMP;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TOPIC;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.VALUE;
+
/**
* The Storm bolt responsible for filtering incoming messages and directing
* each to the downstream bolts responsible for building a Profile.
@@ -132,6 +138,11 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
@Override
public void execute(Tuple input) {
try {
+ LOG.debug("Received message; topic={}, partition={}, offset={},
kafkaTimestamp={}",
+ input.contains(TOPIC.getFieldName()) ?
input.getStringByField(TOPIC.getFieldName()): "unknown",
+ input.contains(PARTITION.getFieldName()) ?
input.getIntegerByField(PARTITION.getFieldName()): "unknown",
+ input.contains(OFFSET.getFieldName()) ?
input.getLongByField(OFFSET.getFieldName()):"unknown",
+ input.contains(TIMESTAMP.getFieldName()) ?
input.getLongByField(TIMESTAMP.getFieldName()): "unknown");
doExecute(input);
} catch (Throwable t) {
@@ -146,7 +157,7 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
private void doExecute(Tuple input) throws ParseException,
UnsupportedEncodingException {
// retrieve the input message
-byte[] data = input.getBinary(0);
+byte[] data = input.getBinaryByField(VALUE.getFieldName());
if(data == null) {
LOG.debug("Received null message. Nothing to do.");
return;
[15/51] [abbrv] metron git commit: METRON-1784: Re-allow remote ssh and scp in Centos full dev (mmiklavc via mmiklavc) closes apache/metron#1204
METRON-1784: Re-allow remote ssh and scp in Centos full dev (mmiklavc via
mmiklavc) closes apache/metron#1204
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4f0b608d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4f0b608d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4f0b608d
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 4f0b608d77ebed3025ba0accedec440b3270cb18
Parents: 08fe1cf
Author: mmiklavc
Authored: Wed Sep 26 16:34:13 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Sep 26 16:34:13 2018 -0600
--
.../roles/enable-remote-ssh/defaults/main.yml | 18 +++
.../roles/enable-remote-ssh/tasks/main.yml | 33
.../development/centos6/ansible/playbook.yml| 5 +--
3 files changed, 54 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
--
diff --git
a/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
new file mode 100644
index 000..f176095
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+sshd_config_file: /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
--
diff --git a/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
new file mode 100644
index 000..be83aa2
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Comment password auth no
+ lineinfile:
+dest: "{{ sshd_config_file }}"
+regexp: "^PasswordAuthentication no"
+line: "#PasswordAuthentication no"
+
+- name: Uncomment password auth yes
+ lineinfile:
+dest: "{{ sshd_config_file }}"
+regexp: "^#PasswordAuthentication yes"
+line: "PasswordAuthentication yes"
+
+- name: Restart service sshd, in all cases
+ service:
+name: sshd
+state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/development/centos6/ansible/playbook.yml
--
diff --git a/metron-deployment/development/centos6/ansible/playbook.yml
b/metron-deployment/development/centos6/ansible/playbook.yml
index fa14677..562fb1d 100644
--- a/metron-deployment/development/centos6/ansible/playbook.yml
+++ b/metron-deployment/development/centos6/ansible/playbook.yml
@@ -17,7 +17,8 @@
---
- hosts: all
roles:
-- role: libselinux-python
-- role: enable-swap
+- { role: libselinux-python, tags: libselinux-python }
+- { role: enable-swap, tags: enable-swap }
+- { role: enable-remote-ssh, tags: enable-remote-ssh }
- include: ../../../ansible/playbooks/metron_full_install.yml
[36/51] [abbrv] metron git commit: METRON-1761 Allow a grok statement to be applied to each line in a file. (ottobackwards) closes apache/metron#1184
METRON-1761 Allow a grok statement to be applied to each line in a file.
(ottobackwards) closes apache/metron#1184
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d923cde
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d923cde
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d923cde
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3d923cde8711dcc463c2d27cb2b67275b9172112
Parents: 060d17e
Author: ottobackwards
Authored: Wed Oct 10 11:31:31 2018 -0400
Committer: otto
Committed: Wed Oct 10 11:31:31 2018 -0400
--
.../src/main/sample/patterns/test | 1 +
metron-platform/metron-parsers/README.md| 10 +-
.../message-parser-implementation-notes.md | 57 +
.../parsers/DefaultMessageParserResult.java | 76 +++
.../org/apache/metron/parsers/GrokParser.java | 156 ++---
.../apache/metron/parsers/bolt/ParserBolt.java | 57 +++--
.../parsers/interfaces/MessageParser.java | 28 ++-
.../parsers/interfaces/MessageParserResult.java | 48
.../parsers/websphere/GrokWebSphereParser.java | 217 +--
.../apache/metron/parsers/GrokParserTest.java | 1 +
.../metron/parsers/MultiLineGrokParserTest.java | 146 +
.../MultiLineWithErrorsGrokParserTest.java | 146 +
.../metron/parsers/SampleGrokParserTest.java| 3 +
.../apache/metron/parsers/SquidParserTest.java | 2 +
.../apache/metron/parsers/YafParserTest.java| 3 +
.../metron/parsers/bolt/ParserBoltTest.java | 15 +-
.../parsers/integration/ParserDriver.java | 2 +-
.../websphere/GrokWebSphereParserTest.java | 13 --
.../test/resources/logData/multi_elb_log.txt| 10 +
.../logData/multi_elb_with_errors_log.txt | 13 ++
20 files changed, 824 insertions(+), 180 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-integration-test/src/main/sample/patterns/test
--
diff --git
a/metron-platform/metron-integration-test/src/main/sample/patterns/test
b/metron-platform/metron-integration-test/src/main/sample/patterns/test
index a88a255..ebbf9c4 100644
--- a/metron-platform/metron-integration-test/src/main/sample/patterns/test
+++ b/metron-platform/metron-integration-test/src/main/sample/patterns/test
@@ -1,2 +1,3 @@
YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T
]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}
YAF_DELIMITED
%{NUMBER:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:protocol}\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\|%{SPACE:UNWANTED}%{INT:ip_src_port}\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
+ELBACCESSLOGS %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb}
%{IP:clientip}:%{INT:clientport} (?:(%{IP:backendip}:?:%{INT:backendport})|-)
%{NUMBER:request_processing_time} %{NUMBER:backend_processing_time}
%{NUMBER:response_processing_time} (?:-|%{INT:elb_status_code})
(?:-|%{INT:backend_status_code}) %{INT:received_bytes} %{INT:sent_bytes}
\"(?:-|(?:%{WORD:verb}
%{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{URIPATH:path}(?:%{URIPARAM:params})?)?(?:
HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}))\"
\"(?:-|%{DATA:user_agent})\" (?:-|%{NOTSPACE:ssl_cipher})
(?:-|%{NOTSPACE:ssl_protocol})
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md
b/metron-platform/metron-parsers/README.md
index 381b0ee..cfcf6ed 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -29,10 +29,12 @@ There are two general types types of parsers:
* Grok parser: `org.apache.metron.parsers.GrokParser` with possible
`parserConfig` entries of
* `grokPath` : The path in HDFS (or in the Jar) to the grok statement
* `patternLabel` : The pattern label to use from the grok statement
+* `multiLine` : The raw data passed in should be handled as a long with
multiple lines, with each line to be parsed separately. This setting's valid
values are 'true' or
[51/51] [abbrv] metron git commit: Revert "METRON-1801 Allow Customization of Elasticsearch Document ID (nickwallen) closes apache/metron#1218"
Revert "METRON-1801 Allow Customization of Elasticsearch Document ID
(nickwallen) closes apache/metron#1218"
This reverts commit 90c5e1d2896b8a95bd6c928e3dab96f1fd85167c.
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e037eda
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e037eda
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e037eda
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0e037edad913955d3b6754ca9cf42b329cd84160
Parents: 0ba1802
Author: Nick Allen
Authored: Tue Oct 23 15:01:04 2018 -0400
Committer: Nick Allen
Committed: Tue Oct 23 15:01:04 2018 -0400
--
Upgrading.md| 23 +--
.../METRON/CURRENT/configuration/metron-env.xml | 9 --
.../CURRENT/package/scripts/metron_service.py | 5 -
.../package/scripts/params/params_linux.py | 1 -
.../METRON/CURRENT/themes/metron_theme.json | 10 --
metron-platform/metron-elasticsearch/README.md | 37 +
metron-platform/metron-elasticsearch/pom.xml| 17 +++
.../dao/ElasticsearchRetrieveLatestDao.java | 147 +++
.../dao/ElasticsearchSearchDao.java | 28 +---
.../elasticsearch/utils/ElasticsearchUtils.java | 100 -
.../writer/ElasticsearchWriter.java | 39 ++---
.../elasticsearch/dao/ElasticsearchDaoTest.java | 10 +-
.../ElasticsearchSearchIntegrationTest.java | 20 +--
.../indexing/dao/SearchIntegrationTest.java | 21 ++-
.../integration/SolrSearchIntegrationTest.java | 19 ---
15 files changed, 145 insertions(+), 341 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/Upgrading.md
--
diff --git a/Upgrading.md b/Upgrading.md
index c3b2a0f..2124ac5 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -16,22 +16,9 @@ See the License for the specific language governing
permissions and
limitations under the License.
-->
# Upgrading
-
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
-## 0.6.0 to 0.6.1
-
-### [METRON-1801 Allow Customization of Elasticsearch Document
ID](https://issues.apache.org/jira/browse/METRON-1801)
-
-A global property named `es.document.id` was added to define the field from
which the document ID is set when a message is indexed by Elasticsearch. To
allow Elasticsearch to define its own document id, this property should be set
to a blank or empty string. The client will not set the document ID and
Elasticsearch will define its own. In most cases allowing Elasticsearch to
define the document ID is the most performant option. This is now the default
behavior.
-
-Metron versions 0.6.0 and earlier defined the document ID using the Metron
GUID, which is a randomized UUID using Java's `UUID.randomUUID()`. Using a
randomized UUID can negatively impact Elasticsearch indexing performance. To
maintain backwards compatibility with legacy versions of Metron use the
following global property setting.
-
-```
-es.document.id = guid
-```
-
## 0.4.2 to 0.5.0
### [METRON-941: native PaloAlto parser corrupts message when having a comma
in the payload](https://issues.apache.org/jira/browse/METRON-941)
@@ -102,7 +89,7 @@ For a more detailed description, please see
metron-platform/metron-elasticsearch
### Description
-In the 0.4.2 release,
+In the 0.4.2 release,
## 0.3.1 to 0.4.0
@@ -120,7 +107,7 @@ This effectively limits the build environment to Docker
supported [platforms](ht
Description
As of 0.3.0 the indexing configuration
-* Is held in the enrichment configuration for a sensor
+* Is held in the enrichment configuration for a sensor
* Has properties which control every writers (i.e. HDFS, solr or
elasticsearch).
In the 0.3.1 release, this configuration has been broken out
@@ -149,7 +136,7 @@ You would create a file to configure each writer for sensor
`foo` called `$METRO
"batchSize" : 100,
"enabled" : true
},
- "hdfs" : {
+ "hdfs" : {
"index" : "foo",
"batchSize" : 100,
"enabled" : true
@@ -164,7 +151,7 @@ You would create a file to configure each writer for sensor
`foo` called `$METRO
As of 0.3.0, threat triage rules were defined as a simple Map associating a
Stellar expression with a score.
As of 0.3.1, due to the fact that there may be many threat triage rules, we
have made the rules more complex.
To help organize these, we have made the threat triage objects in their own
right that contain optional name and optional comment fields.
-
+
This essentially makes the risk level rules slightly more complex. The format
goes from:
```
"riskLevelRules" : {
@@ -182,7 +169,7 @@ to:
}
]
```
-
+
Migration
[29/51] [abbrv] metron git commit: METRON-1807 Auto populate the recommended values to some of the metron config parameters (MohanDV via merrimanr) closes apache/metron#1227
METRON-1807 Auto populate the recommended values to some of the metron config
parameters (MohanDV via merrimanr) closes apache/metron#1227
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/269b91d0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/269b91d0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/269b91d0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 269b91d015c67b0ae49a78e5c457408e1b431c25
Parents: 891bf49
Author: MohanDV
Authored: Mon Oct 8 11:20:53 2018 -0500
Committer: rmerriman
Committed: Mon Oct 8 11:20:53 2018 -0500
--
.../METRON/CURRENT/configuration/metron-env.xml | 12
.../common-services/METRON/CURRENT/service_advisor.py | 10 ++
2 files changed, 18 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index cdef7cf..81dda6c 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -130,17 +130,21 @@
true
-
+
storm_rest_addr
Storm Rest Server Address
URL of Storm UI (http://storm.ui.hostname:8744). If no
protocol is provided, http is assumed.
-
+
+true
+
-
+
zeppelin_server_url
Zeppelin Server Address
URL of Zeppelin
(zeppelin.server.addr:zeppelin.server.port)
-
+
+true
+
metron_principal_name
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index b008fc0..36cec7c 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -158,6 +158,16 @@ class
METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
putMetronEnvProperty = self.putProperty(configurations,
"metron-env", services)
putMetronEnvProperty("zeppelin_server_url", zeppelinServerUrl)
+#Suggest Zookeeper quorum
+if "solr-cloud" in services["configurations"]:
+zookeeperHost = self.getComponentHostNames(services, "ZOOKEEPER",
"ZOOKEEPER_SERVER")[0]
+zookeeperClientPort =
services["configurations"]["zoo.cfg"]["properties"]["clientPort"]
+solrZkDir =
services["configurations"]["solr-cloud"]["properties"]["solr_cloud_zk_directory"]
+solrZookeeperUrl = zookeeperHost + ":" + zookeeperClientPort +
solrZkDir
+putMetronEnvProperty = self.putProperty(configurations,
"metron-env", services)
+putMetronEnvProperty("solr_zookeeper_url", solrZookeeperUrl)
+
+
def validateSTORMSiteConfigurations(self, properties, recommendedDefaults,
configurations, services, hosts):
# Determine if the cluster is secured
is_secured = self.isSecurityEnabled(services)
[17/51] [abbrv] metron git commit: METRON-1758 Add support for Ansible 2.6 in dev (JonZeolla via jonzeolla) closes apache/metron#1179
METRON-1758 Add support for Ansible 2.6 in dev (JonZeolla via jonzeolla) closes
apache/metron#1179
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1723a0ea
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1723a0ea
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1723a0ea
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1723a0eae57adcfc4a2efe2422f52fbed4c237e7
Parents: fbabfca
Author: JonZeolla
Authored: Mon Oct 1 08:41:25 2018 -0400
Committer: jonzeolla
Committed: Mon Oct 1 08:41:25 2018 -0400
--
metron-deployment/amazon-ec2/README.md | 2 +-
metron-deployment/amazon-ec2/ansible.cfg| 1 +
metron-deployment/amazon-ec2/playbook.yml | 4 --
metron-deployment/ansible.cfg | 23 --
.../0_minimum_ansible_version.py| 45
.../ansible/playbooks/metron_full_install.yml | 16 +++
.../ambari_common/tasks/ambari-repo-ubuntu.yml | 3 +-
.../ansible/roles/ambari_common/tasks/main.yml | 13 +++---
.../roles/ambari_common/tasks/nodejs.yml| 6 ++-
.../ansible/roles/enable-swap/tasks/main.yml| 3 +-
.../ansible/roles/java_jdk/tasks/main.yml | 6 +--
.../ansible/roles/metron-builder/tasks/main.yml | 6 +--
.../roles/metron-packages/tasks/main.yml| 12 ++
metron-deployment/development/centos6/README.md | 5 +--
.../development/centos6/Vagrantfile | 2 +-
.../development/centos6/ansible.cfg | 2 +-
.../development/centos6/ansible/playbook.yml| 2 +-
.../development/fastcapa/centos-7.1/ansible.cfg | 2 +-
.../development/fastcapa/centos-7.4/ansible.cfg | 2 +-
.../development/ubuntu14/README.md | 5 +--
.../development/ubuntu14/Vagrantfile| 2 +-
.../development/ubuntu14/ansible.cfg| 2 +-
.../development/ubuntu14/ansible/playbook.yml | 2 +-
23 files changed, 84 insertions(+), 82 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/README.md
--
diff --git a/metron-deployment/amazon-ec2/README.md
b/metron-deployment/amazon-ec2/README.md
index b2efc9e..ed1c73b 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -38,7 +38,7 @@ Getting Started
The host used to deploy Apache Metron will need the following software tools
installed. The following versions are known to work as of the time of this
writing, but by no means are these the only working versions.
- - Ansible 2.0.0.2, 2.2.2.0, or 2.5.0
+ - Ansible 2.4.0+
- Python 2.7.11
- Maven 3.3.9
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/ansible.cfg
--
diff --git a/metron-deployment/amazon-ec2/ansible.cfg
b/metron-deployment/amazon-ec2/ansible.cfg
index 0fe3343..4291cc8 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -22,6 +22,7 @@ pipelining = True
remote_user = centos
forks = 20
log_path = ./ansible.log
+callback_plugins = ../ansible/callback_plugins
# fix for "ssh throws 'unix domain socket too long' " problem
#[ssh_connection]
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/playbook.yml
--
diff --git a/metron-deployment/amazon-ec2/playbook.yml
b/metron-deployment/amazon-ec2/playbook.yml
index cb1e326..4e8880a 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -21,10 +21,6 @@
- hosts: localhost
vars_files:
- conf/defaults.yml
- pre_tasks:
-- name: Verify Ansible Version
- fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0 or 2.5.0, current
version is {{ ansible_version }}"
- when: "ansible_version.full | version_compare('2.2.2.0', '!=') and
ansible_version.full | version_compare('2.0.0.2', '!=') and
ansible_version.full | version_compare('2.5.0', '!=')"
tasks:
- include: tasks/create-keypair.yml
- include: tasks/create-vpc.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible.cfg
--
diff --git a/metron-deployment/ansible.cfg b/metron-deployment/ansible.cfg
deleted file mode 100644
index 9b3916b..000
--- a/metron-deployment/ansible.cfg
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding
[34/51] [abbrv] metron git commit: METRON-1812: Fix dependencies_with_url.csv (mmiklavc via mmiklavc) closes apache/metron#1230
METRON-1812: Fix dependencies_with_url.csv (mmiklavc via mmiklavc) closes apache/metron#1230 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1f1b9cdd Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1f1b9cdd Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1f1b9cdd Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 1f1b9cdd3cef7557ff68b7ab33693161b142ef1d Parents: 747220f Author: mmiklavc Authored: Tue Oct 9 14:51:57 2018 -0600 Committer: Michael Miklavcic Committed: Tue Oct 9 14:51:57 2018 -0600 -- dependencies_with_url.csv | 32 ++-- 1 file changed, 14 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/1f1b9cdd/dependencies_with_url.csv -- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 53977f3..fb6c03c 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -256,12 +256,8 @@ io.dropwizard.metrics:metrics-json:jar:3.1.5:compile,ASLv2,https://github.com/dr io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2, io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2, -<<< HEAD io.netty:netty-all:jar:4.1.17.Final:compile,ASLv2, -=== io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2, -io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/ ->>> apache/master io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.9.9.Final:compile,Apache License, Version 2.0,http://netty.io/ @@ -472,20 +468,20 @@ org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,ht com.github.ben-manes.caffeine:caffeine:jar:2.6.2:compile,ASLv2,https://github.com/ben-manes/caffeine/blob/v2.6.2/LICENSE com.google.code.gson:gson:jar:2.2:compile,ASLv2,https://github.com/google/gson com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson - org.codehaus.plexus:plexus-classworlds:jar:2.4:compile - org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile - org.codehaus.plexus:plexus-interpolation:jar:1.14:compile - org.codehaus.plexus:plexus-utils:jar:2.0.7:compile - org.jsoup:jsoup:jar:1.6.1:compile - org.sonatype.aether:aether-api:jar:1.12:compile - org.sonatype.aether:aether-connector-file:jar:1.12:compile - org.sonatype.aether:aether-connector-wagon:jar:1.12:compile - org.sonatype.aether:aether-impl:jar:1.12:compile - org.sonatype.aether:aether-spi:jar:1.12:compile - org.sonatype.aether:aether-util:jar:1.12:compile - org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile - org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile - org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile +org.codehaus.plexus:plexus-classworlds:jar:2.4:compile +org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile +org.codehaus.plexus:plexus-interpolation:jar:1.14:compile +org.codehaus.plexus:plexus-utils:jar:2.0.7:compile +org.jsoup:jsoup:jar:1.6.1:compile +org.sonatype.aether:aether-api:jar:1.12:compile +org.sonatype.aether:aether-connector-file:jar:1.12:compile +org.sonatype.aether:aether-connector-wagon:jar:1.12:compile +org.sonatype.aether:aether-impl:jar:1.12:compile +org.sonatype.aether:aether-spi:jar:1.12:compile +org.sonatype.aether:aether-util:jar:1.12:compile +org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile +org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile +org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
[22/51] [abbrv] metron git commit: METRON-1804 Update version to 0.6.1 (justinleet) closes apache/metron#1220
METRON-1804 Update version to 0.6.1 (justinleet) closes apache/metron#1220 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b872fdcd Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b872fdcd Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b872fdcd Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: b872fdcd0a54f41165e93d0e3635635a09e61ef8 Parents: ebdaf5f Author: justinleet Authored: Wed Oct 3 15:25:44 2018 -0400 Committer: leet Committed: Wed Oct 3 15:25:44 2018 -0400 -- metron-analytics/metron-maas-common/pom.xml| 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml| 2 +- metron-analytics/metron-profiler-common/pom.xml| 2 +- metron-analytics/metron-profiler-repl/pom.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml | 2 +- metron-analytics/metron-profiler-storm/pom.xml | 2 +- metron-analytics/metron-statistics/pom.xml | 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- metron-deployment/ansible/playbooks/docker_probe_install.yml | 2 +- .../ansible/roles/ambari_master/defaults/main.yml | 2 +- .../development/centos6/ansible/inventory/group_vars/all | 2 +- .../development/ubuntu14/ansible/inventory/group_vars/all | 2 +- metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml | 4 ++-- metron-deployment/packaging/ambari/metron-mpack/pom.xml| 4 ++-- .../ambari/metron-mpack/src/main/resources/mpack.json | 2 +- metron-deployment/packaging/docker/deb-docker/pom.xml | 4 ++-- metron-deployment/packaging/docker/rpm-docker/pom.xml | 4 ++-- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/package-lock.json | 2 +- metron-interface/metron-alerts/package.json| 2 +- metron-interface/metron-alerts/pom.xml | 2 +- metron-interface/metron-config/package-lock.json | 2 +- metron-interface/metron-config/package.json| 2 +- metron-interface/metron-config/pom.xml | 2 +- metron-interface/metron-config/scripts/package.json| 2 +- metron-interface/metron-rest-client/pom.xml| 2 +- metron-interface/metron-rest/pom.xml | 2 +- .../metron-rest/src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md| 4 ++-- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 2 +- metron-platform/metron-common/src/main/scripts/cluster_info.py | 2 +- metron-platform/metron-data-management/pom.xml | 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml| 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml| 2 +- metron-platform/metron-integration-test/pom.xml| 2 +- metron-platform/metron-job/pom.xml | 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsers/pom.xml | 2 +- metron-platform/metron-pcap-backend/pom.xml| 2 +- .../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml| 2 +- metron-platform/metron-solr/pom.xml| 2 +- metron-platform/metron-storm-kafka-override/pom.xml| 2 +- metron-platform/metron-storm-kafka/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- metron-platform/metron-zookeeper/pom.xml | 2 +- metron-platform/pom.xml| 2 +- metron-stellar/pom.xml | 2 +- metron-stellar/stellar-3rd-party-example/pom.xml | 2 +-
[24/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 2f83921..54b5b64 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -56,7 +56,7 @@ public class SolrUpdateDao implements UpdateDao {
}
@Override
- public void update(Document update, Optional rawIndex) throws
IOException {
+ public Document update(Document update, Optional rawIndex) throws
IOException {
Document newVersion = update;
// Handle any case where we're given comments in Map form, instead of raw
String
Object commentsObj = update.getDocument().get(COMMENTS_FIELD);
@@ -79,10 +79,11 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+return newVersion;
}
@Override
- public void batchUpdate(Map> updates) throws
IOException {
+ public Map> batchUpdate(Map> updates) throws IOException {
// updates with a collection specified
Map> solrCollectionUpdates = new
HashMap<>();
Set collectionsUpdated = new HashSet<>();
@@ -117,18 +118,20 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+return updates;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws
IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws
IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(),
request.getSensorType());
-addCommentToAlert(request, latest);
+return addCommentToAlert(request, latest);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document
latest) throws IOException {
-if (latest == null) {
- return;
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document
latest) throws IOException {
+if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to add comment. Document
with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -149,21 +152,22 @@ public class SolrUpdateDao implements UpdateDao {
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentStrs);
-update(newVersion, Optional.empty());
+return update(newVersion, Optional.empty());
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request)
throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(),
request.getSensorType());
-removeCommentFromAlert(request, latest);
+return removeCommentFromAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document
latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request,
Document latest)
throws IOException {
-if (latest == null) {
- return;
+if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to remove comment. Document
with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -171,8 +175,8 @@ public class SolrUpdateDao implements UpdateDao {
.get(COMMENTS_FIELD);
// Can't remove anything if there's nothing there
if (commentMap == null) {
- LOG.debug("Provided alert had no comments to be able to remove from");
- return;
+ throw new IOException(String.format("Unable to remove comment. Document
with guid %s has no comments.",
+ request.getGuid()));
}
List> originalComments = new ArrayList<>(commentMap);
List comments = new ArrayList<>();
@@ -186,7 +190,7 @@ public class SolrUpdateDao implements UpdateDao {
.collect(Collectors.toList());
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson);
-update(newVersion, Optional.empty());
+return update(newVersion, Optional.empty());
}
public void convertCommentsToRaw(Map source) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
--
diff --git
[30/51] [abbrv] metron git commit: METRON-1792 Simplify Profile Definitions in Integration Tests (nickwallen) closes apache/metron#1211
METRON-1792 Simplify Profile Definitions in Integration Tests (nickwallen)
closes apache/metron#1211
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5bfc08c5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5bfc08c5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5bfc08c5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 5bfc08c57f1129b7d185ac7257197775ed3bdb5e
Parents: 269b91d
Author: nickwallen
Authored: Mon Oct 8 18:32:30 2018 -0400
Committer: nickallen
Committed: Mon Oct 8 18:32:30 2018 -0400
--
.../zookeeper/event-time-test/profiler.json | 19
.../processing-time-test/profiler.json | 11 --
.../zookeeper/profile-with-stats/profiler.json | 12 --
.../integration/ConfigUploadComponent.java | 31 --
.../integration/ProfilerIntegrationTest.java| 109 ---
.../ZKConfigurationsCacheIntegrationTest.java | 5 +-
.../src/test/resources/profiler/profiler.json | 19
7 files changed, 137 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
--
diff --git
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
deleted file mode 100644
index 534b7c6..000
---
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "timestampField": "timestamp",
- "profiles": [
-{
- "profile": "count-by-ip",
- "foreach": "ip_src_addr",
- "init": { "count": 0 },
- "update": { "count" : "count + 1" },
- "result": "count"
-},
-{
- "profile": "total-count",
- "foreach": "'total'",
- "init": { "count": 0 },
- "update": { "count": "count + 1" },
- "result": "count"
-}
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
--
diff --git
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
deleted file mode 100644
index e75ec0f..000
---
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "profiles": [
-{
- "profile": "processing-time-test",
- "foreach": "ip_src_addr",
- "init": { "counter": "0" },
- "update": { "counter": "counter + 1" },
- "result": "counter"
-}
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
--
diff --git
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
deleted file mode 100644
index 083e73f..000
---
a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "profiles": [
-{
- "profile": "profile-with-stats",
- "foreach": "'global'",
- "init": { "stats": "STATS_INIT()" },
- "update": { "stats": "STATS_ADD(stats, 1)" },
- "result": "stats"
-}
- ],
- "timestampField": "timestamp"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
--
diff --git
a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
index 70487a0..eae3c52 100644
---
a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
+++
b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
@@ -19,12 +19,15 @@
*/
package
[32/51] [abbrv] metron git commit: METRON-1809 Support Column Oriented Input with Batch Profiler (nickwallen) closes apache/metron#1229
METRON-1809 Support Column Oriented Input with Batch Profiler (nickwallen)
closes apache/metron#1229
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3467ffdb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3467ffdb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3467ffdb
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3467ffdb13aad22ab1e4065a0b6715b99371aa23
Parents: e482366
Author: nickwallen
Authored: Tue Oct 9 13:55:45 2018 -0400
Committer: nickallen
Committed: Tue Oct 9 13:55:45 2018 -0400
--
.../metron-profiler-spark/README.md | 53 +++--
.../metron/profiler/spark/BatchProfiler.java| 21 ++--
.../profiler/spark/BatchProfilerConfig.java | 9 +-
.../reader/ColumnEncodedTelemetryReader.java| 84 +
.../profiler/spark/reader/TelemetryReader.java | 43 +++
.../profiler/spark/reader/TelemetryReaders.java | 110 +
.../reader/TextEncodedTelemetryReader.java | 83 +
.../spark/BatchProfilerIntegrationTest.java | 36 +-
.../ColumnEncodedTelemetryReaderTest.java | 118 +++
.../spark/function/reader/IsValidJSON.java | 38 ++
.../function/reader/TelemetryReadersTest.java | 89 ++
.../reader/TextEncodedTelemetryReaderTest.java | 114 ++
12 files changed, 770 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/README.md
--
diff --git a/metron-analytics/metron-profiler-spark/README.md
b/metron-analytics/metron-profiler-spark/README.md
index df143f1..5ed5f4b 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -162,13 +162,13 @@ ${SPARK_HOME}/bin/spark-submit \
The Batch Profiler accepts the following arguments when run from the command
line as shown above. All arguments following the Profiler jar are passed to
the Profiler. All argument preceeding the Profiler jar are passed to Spark.
-| Argument | Description
-|--- |---
-| -p, --profiles | Path to the profile definitions.
-| -c, --config | Path to the profiler properties file.
-| -g, --globals| Path to the Stellar global config file.
-| -r, --reader | Path to properties for the DataFrameReader.
-| -h, --help | Print the help text.
+| Argument | Description
+|---|---
+| [`-p`, `--profiles`](#--profiles) | Path to the profile definitions.
+| [`-c`, `--config`](#--config) | Path to the profiler properties file.
+| [`-g`, `--globals`](#--globals) | Path to the Stellar global config
file.
+| [`-r`, `--reader`](#--reader) | Path to properties for the
DataFrameReader.
+| `-h`, `--help`| Print the help text.
`--profiles`
@@ -234,6 +234,28 @@ The Profiler can consume archived telemetry stored in a
variety of input formats
1. If additional options are required for your input format, then use the
[`--reader`](#--reader) command-line argument when launching the Batch Profiler
as [described here](#advanced-usage).
+ Common Formats
+
+The following examples highlight the configuration values needed to read
telemetry stored in common formats. These values should be defined in the
Profiler properties (see [`--config`](#--config)).
+
+# JSON
+```
+profiler.batch.input.reader=json
+profiler.batch.input.path=/path/to/json/
+```
+
+# [Apache ORC](https://orc.apache.org/)
+```
+profiler.batch.input.reader=orc
+profiler.batch.input.path=/path/to/orc/
+```
+
+# [Apache Parquet](http://parquet.apache.org/)
+```
+profiler.batch.input.reader=parquet
+profiler.batch.input.path=/path/to/parquet/
+```
+
## Configuring the Profiler
@@ -244,6 +266,7 @@ You can store both settings for the Profiler along with
settings for Spark in th
| Setting
| Description
|---
|---
| [`profiler.batch.input.path`](#profilerbatchinputpath)
| The path to the input data read by the Batch Profiler.
+| [`profiler.batch.input.reader`](#profilerbatchinputreader)
| The telemetry reader used to read the input data.
| [`profiler.batch.input.format`](#profilerbatchinputformat)
| The format of the input data read by the Batch Profiler.
| [`profiler.batch.input.begin`](#profilerbatchinputend)
| Only messages with a timestamp after this will be profiled.
|
[13/51] [abbrv] metron git commit: METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185
METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch
(nickwallen) closes apache/metron#1185
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e
Parents: 0d34e77
Author: nickwallen
Authored: Fri Sep 21 15:31:24 2018 -0400
Committer: nickallen
Committed: Fri Sep 21 15:31:24 2018 -0400
--
.../ansible/roles/ambari_config/vars/single_node_vm.yml | 4 ++--
.../development/centos6/ansible/inventory/group_vars/all | 1 +
.../development/ubuntu14/ansible/inventory/group_vars/all| 1 +
.../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml | 8
4 files changed, 8 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 962726c..2ec956a 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -92,7 +92,7 @@ configurations:
# Storm expects ambari metrics to be available in 2.6. We do *not*
install ambari metrics in full-dev, so we need to revert to the old consumer
storm.cluster.metrics.consumer.register: '[{"class":
"org.apache.storm.metric.LoggingMetricsConsumer"}]'
topology.metrics.consumer.register: '[{"class":
"org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1,
"whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency",
"__receive\\.population$", "__sendqueue\\.population$", "__execute-count",
"__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$",
"memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
- # Storm expects ambari metrics to be available in 2.6 and ambari metrics
pulls data via JMX, but since we don't use ambari metrics here, we don't have
the javaagent around to use and thus that must be removed from nimbus,
supervisor and worker properties
+ # Storm expects ambari metrics to be available in 2.6 and ambari metrics
pulls data via JMX, but since we don't use ambari metrics here, we don't have
the javaagent around to use and thus that must be removed from nimbus,
supervisor and worker properties
nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
@@ -117,7 +117,7 @@ configurations:
required_configurations:
- metron-env:
storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
- es_hosts: "{{ groups.search | join(',') }}"
+ es_hosts: "{{ elasticsearch_hosts }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
solr_zookeeper_url: "{{ groups.search[0] }}:9983"
- metron-rest-env:
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all
--
diff --git
a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index dcb7b42..550eb8f 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
threat_intel_bulk_load: False
+elasticsearch_hosts: "{{ groups.search | join(',') }}"
# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
--
diff --git
a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index ef9f705..1338d7c 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
[11/51] [abbrv] metron git commit: METRON-1709 Add controls to start / stop the PCAP topology from Ambari. (MohanDV via nickwallen) closes apache/metron#1201
METRON-1709 Add controls to start / stop the PCAP topology from Ambari.
(MohanDV via nickwallen) closes apache/metron#1201
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/15194c3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/15194c3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/15194c3b
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 15194c3b4e7d9d9d161b0ddfa556395b34cfef7c
Parents: 3372aa3
Author: MohanDV
Authored: Thu Sep 20 11:10:07 2018 -0400
Committer: nickallen
Committed: Thu Sep 20 11:10:07 2018 -0400
--
.../roles/ambari_config/vars/single_node_vm.yml | 3 +-
.../roles/ambari_config/vars/small_cluster.yml | 3 +-
.../METRON/CURRENT/role_command_order.json | 9 +-
.../common-services/METRON/CURRENT/metainfo.xml | 39
.../package/scripts/params/params_linux.py | 4 +
.../package/scripts/params/status_params.py | 1 +
.../CURRENT/package/scripts/pcap_commands.py| 201 +++
.../CURRENT/package/scripts/pcap_master.py | 105 ++
.../CURRENT/package/scripts/rest_commands.py| 43
.../CURRENT/package/scripts/rest_master.py | 7 -
.../METRON/CURRENT/service_advisor.py | 5 +
11 files changed, 365 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index a3c643b..962726c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -32,6 +32,7 @@ es_master: [ES_MASTER]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -40,7 +41,7 @@ metron_alerts_ui: [METRON_ALERTS_UI]
metron_components: >
{{ hadoop_master | union(zookeeper_master) | union(storm_master) |
union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
- union(storm_slave) | union(kafka_broker) | union(hbase_slave) |
union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
+ union(storm_slave) | union(kafka_broker) | union(hbase_slave) |
union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
union(metron_pcap) |
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest)
| union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
cluster_name: "metron_cluster"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
index 218e267..818b5f3 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -32,6 +32,7 @@ es_slave: [ES_SLAVE]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -45,7 +46,7 @@ master_2_components: "{{ zookeeper_master |
union(storm_master) | union(spark_ma
master_2_host:
- "{{groups.ambari_slave[1]}}"
metron_components: >
-{{ metron_indexing | union(metron_profiler) |
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) |
union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) |
union(storm_slave) |
+{{ metron_indexing | union(metron_profiler) | union(metron_pcap)
|union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) |
union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) |
union(storm_slave) |
union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
- "{{ groups.metron[0] }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
--
diff --git
[14/51] [abbrv] metron git commit: METRON-1787 Input Time Constraints for Batch Profiler (nickwallen) closes apache/metron#1209
METRON-1787 Input Time Constraints for Batch Profiler (nickwallen) closes
apache/metron#1209
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa3be8d3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa3be8d3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa3be8d3
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: fa3be8d32ccadcd11edad046cbd063cec3a20624
Parents: 1545978
Author: nickwallen
Authored: Wed Sep 26 18:13:30 2018 -0400
Committer: nickallen
Committed: Wed Sep 26 18:13:30 2018 -0400
--
.../clock/EventTimeOnlyClockFactory.java| 58
.../clock/EventTimeOnlyClockFactoryTest.java| 61 +
.../metron-profiler-spark/README.md | 19 +++
.../metron/profiler/spark/BatchProfiler.java| 38 +-
.../profiler/spark/BatchProfilerConfig.java | 6 +-
.../metron/profiler/spark/TimestampParser.java | 55
.../spark/function/MessageRouterFunction.java | 106 +--
.../spark/BatchProfilerIntegrationTest.java | 58 +++-
.../profiler/spark/TimestampParserTest.java | 67 ++
.../function/MessageRouterFunctionTest.java | 133 +--
10 files changed, 579 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
--
diff --git
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
new file mode 100644
index 000..2f9ca7c
--- /dev/null
+++
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.clock;
+
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+
+import java.io.Serializable;
+
+/**
+ * Creates a {@link Clock} based on the profiler configuration. This should
+ * be used in cases where only event time is accceptable.
+ *
+ * If the Profiler is configured to use event time, a {@link
EventTimeClock} will
+ * be created. Otherwise, an {@link IllegalStateException} is thrown.
+ */
+public class EventTimeOnlyClockFactory implements ClockFactory, Serializable {
+
+ /**
+ * If the Profiler is configured to use event time, a {@link EventTimeClock}
is created.
+ * Otherwise, an {@link IllegalArgumentException} is thrown.
+ *
+ * @param config The profiler configuration.
+ * @return The appropriate Clock based on the profiler configuration.
+ * @throws IllegalStateException If the profiler configuration is set to
system time.
+ */
+ @Override
+ public Clock createClock(ProfilerConfig config) {
+Clock clock;
+
+boolean isEventTime = config.getTimestampField().isPresent();
+if(isEventTime) {
+ String timestampField = config.getTimestampField().get();
+ clock = new EventTimeClock(timestampField);
+
+} else {
+ throw new IllegalStateException("Expected profiler to use event time.");
+}
+
+return clock;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
--
diff --git
a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
new file mode 100644
index 000..f1d4114
--- /dev/null
+++
[49/51] [abbrv] metron git commit: METRON-1839 Install Elasticsearch MPack Step in Ansible Not Idempotent (nickwallen) closes apache/metron#1244
METRON-1839 Install Elasticsearch MPack Step in Ansible Not Idempotent
(nickwallen) closes apache/metron#1244
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/739782ab
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/739782ab
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/739782ab
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 739782ab71f6e571ff459d1844a77b533abbdbea
Parents: 0c0602c
Author: nickwallen
Authored: Mon Oct 22 15:39:25 2018 -0400
Committer: nickallen
Committed: Mon Oct 22 15:39:25 2018 -0400
--
metron-deployment/ansible/roles/ambari_master/defaults/main.yml | 1 +
.../ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml| 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/739782ab/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
--
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
index 8bae8ce..f6d76e5 100644
--- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -19,5 +19,6 @@ ambari_server_mem: 2048
ambari_mpack_version: 0.6.1.0
metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
metron_mpack_path: "{{ playbook_dir
}}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
+elasticsearch_mpack_version: 5.6.2
elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
elasticsearch_mpack_path: "{{ playbook_dir
}}/../../packaging/ambari/elasticsearch-mpack/target/{{
elasticsearch_mpack_name }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/739782ab/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
index 7ce1a13..ea5b0f4 100644
---
a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
+++
b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
@@ -21,6 +21,6 @@
dest: /tmp
- name: Install Elasticsearch MPack on Ambari Host
- shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{
ambari_mpack_version }}.tar.gz
+ shell: "ambari-server install-mpack --mpack=/tmp/{{ elasticsearch_mpack_name
}}"
args:
-creates:
/var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{
ambari_mpack_version }}/addon-services
+creates:
/var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{
elasticsearch_mpack_version }}/addon-services
[12/51] [abbrv] metron git commit: METRON-1786 Pcap Topology Status Incorrect (MohanDV via nickwallen) closes apache/metron#1206
METRON-1786 Pcap Topology Status Incorrect (MohanDV via nickwallen) closes apache/metron#1206 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0d34e77c Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0d34e77c Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0d34e77c Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 0d34e77cdb0c9104c848808800158492620c7d9b Parents: 15194c3 Author: MohanDV Authored: Fri Sep 21 09:41:23 2018 -0400 Committer: nickallen Committed: Fri Sep 21 09:41:23 2018 -0400 -- .../METRON/CURRENT/package/scripts/params/status_params.py | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/0d34e77c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 2c711cf..1ca29b2 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -119,6 +119,8 @@ metron_principal_name = config['configurations']['metron-env']['metron_principal metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab'] # Pcap +metron_pcap_topology = 'pcap' +pcap_input_topic = 'pcap' pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured' pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
[01/51] [abbrv] metron git commit: Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler
Repository: metron Updated Branches: refs/heads/feature/METRON-1090-stellar-assignment 9ce4ba5a9 -> 0e037edad http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec -- diff --cc metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 38b3770,fca0949..4bcef33 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@@ -588,10 -546,8 +588,12 @@@ chkconfig --del metron-management-u chkconfig --del metron-alerts-ui %changelog - * Tue Aug 21 2018 Apache Metron - 0.5.1 -* Thu Aug 30 2018 Apache Metron - 0.5.1 ++* Thu Aug 30 2018 Apache Metron - 0.6.1 + - Update compiled css file name for Alerts UI ++* Tue Aug 21 2018 Apache Metron - 0.6.1 +- Add Profiler for REPL +* Tue Aug 14 2018 Apache Metron - 0.5.1 +- Add Profiler for Spark * Thu Feb 1 2018 Apache Metron - 0.4.3 - Add Solr install script to Solr RPM * Tue Sep 25 2017 Apache Metron - 0.4.2 http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-deployment/packaging/docker/rpm-docker/pom.xml -- http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/pom.xml --
[03/51] [abbrv] metron git commit: Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
--
diff --cc
metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
index fc94afa,000..3f16edd
mode 100644,00..100644
---
a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
+++
b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
@@@ -1,356 -1,0 +1,378 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.storm;
+
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfigurations;
+import org.apache.metron.profiler.MessageDistributor;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.storm.integration.MessageBuilder;
+import org.apache.metron.test.bolt.BaseBoltTest;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseWindowedBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.windowing.TupleWindow;
+import org.json.simple.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
++import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests the ProfileBuilderBolt.
+ */
+public class ProfileBuilderBoltTest extends BaseBoltTest {
+
+ private JSONObject message1;
+ private JSONObject message2;
+ private ProfileConfig profile1;
+ private ProfileConfig profile2;
+ private ProfileMeasurementEmitter emitter;
+ private ManualFlushSignal flushSignal;
+ private ProfileMeasurement measurement;
+
+ @Before
+ public void setup() throws Exception {
+
+message1 = new MessageBuilder()
+.withField("ip_src_addr", "10.0.0.1")
+.withField("value", "22")
+.build();
+
+message2 = new MessageBuilder()
+.withField("ip_src_addr", "10.0.0.2")
+.withField("value", "22")
+.build();
+
+profile1 = new ProfileConfig()
+.withProfile("profile1")
+.withForeach("ip_src_addr")
+.withInit("x", "0")
+.withUpdate("x", "x + 1")
+.withResult("x");
+
+profile2 = new ProfileConfig()
+.withProfile("profile2")
+.withForeach("ip_src_addr")
+.withInit(Collections.singletonMap("x", "0"))
+.withUpdate(Collections.singletonMap("x", "x + 1"))
+.withResult("x");
+
+measurement = new ProfileMeasurement()
+.withEntity("entity1")
+.withProfileName("profile1")
+.withPeriod(1000, 500, TimeUnit.MILLISECONDS)
+.withProfileValue(22);
+
+flushSignal = new ManualFlushSignal();
+flushSignal.setFlushNow(false);
+ }
+
+ /**
+ * The bolt should extract a message and timestamp from a tuple and
+ * pass that to a {@code MessageDistributor}.
+ */
+ @Test
+ public void testExtractMessage() throws Exception {
+
+ProfileBuilderBolt bolt = createBolt();
+
+// create a mock
+MessageDistributor distributor =
[07/51] [abbrv] metron git commit: METRON-1774 Allow user to configure JAAS client in Ambari (nickwallen) closes apache/metron#1192
METRON-1774 Allow user to configure JAAS client in Ambari (nickwallen) closes
apache/metron#1192
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b84c8740
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b84c8740
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b84c8740
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b84c8740db5fbd05f3876aee3cb5532622a6b9dd
Parents: 91c410f
Author: nickwallen
Authored: Tue Sep 18 11:35:13 2018 -0400
Committer: nickallen
Committed: Tue Sep 18 11:35:13 2018 -0400
--
.../configuration/metron-client-jaas-conf.xml | 61
.../common-services/METRON/CURRENT/metainfo.xml | 1 +
.../CURRENT/package/scripts/metron_security.py | 4 +-
.../package/scripts/params/params_linux.py | 2 +
.../package/templates/client_jaas.conf.j2 | 44 --
5 files changed, 66 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
new file mode 100644
index 000..141a5eb
--- /dev/null
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
@@ -0,0 +1,61 @@
+
+
+
+
+
+content
+metron_client_jaas template
+Metron client JAAS configuration
+
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=false
+ storeKey=false
+ useTicketCache=true
+ serviceName="nimbus"
+ principal="{{metron_principal_name}}";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{metron_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{metron_principal_name}}";
+};
+KafkaClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{metron_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="kafka"
+ principal="{{metron_principal_name}}";
+};
+
+
+ content
+ false
+
+
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index f83d93b..644ba97 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -471,6 +471,7 @@
kafka-broker
kafka-env
zeppelin-config
+metron-client-jaas-conf
true
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
index 4f04daf..d9486e8 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
@@ -16,6 +16,7 @@ limitations under the License.
import os.path
from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
from resource_management.core.resources.system import Directory, File
from resource_management.core import global_lock
from resource_management.core.logger import Logger
@@ -46,7 +47,7 @@ def storm_security_setup(params):
)
File(ambari_format('{client_jaas_path}'),
[04/51] [abbrv] metron git commit: Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler
Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cad2f408 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cad2f408 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cad2f408 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: cad2f4086a29f25a5ae2aeaf70d0933776df3681 Parents: fb623f6 02a1280 Author: Nick Allen Authored: Tue Sep 18 10:53:41 2018 -0400 Committer: Nick Allen Committed: Tue Sep 18 10:53:41 2018 -0400 -- .gitignore | 8 + metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml| 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- .../profiler/DefaultMessageDistributor.java |18 +- .../src/test/resources/log4j.properties | 3 + metron-analytics/metron-profiler-repl/pom.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml | 2 +- metron-analytics/metron-profiler-storm/pom.xml | 2 +- .../profiler/storm/ProfileBuilderBolt.java |71 +- .../zookeeper/event-time-test/profiler.json |19 +- .../profiler/storm/ProfileBuilderBoltTest.java |22 + .../integration/ProfilerIntegrationTest.java| 351 +- .../src/test/resources/log4j.properties |10 +- .../src/test/resources/telemetry.json | 100 + metron-analytics/metron-statistics/pom.xml | 2 +- metron-analytics/pom.xml| 2 +- metron-contrib/metron-docker/pom.xml| 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 4 +- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- .../ansible/playbooks/docker_probe_install.yml | 2 +- .../roles/ambari_master/defaults/main.yml | 2 +- .../centos6/ansible/inventory/group_vars/all| 2 +- .../ubuntu14/ansible/inventory/group_vars/all | 2 +- .../ambari/elasticsearch-mpack/pom.xml | 4 +- .../packaging/ambari/metron-mpack/pom.xml | 4 +- .../metron-mpack/src/main/resources/mpack.json | 2 +- .../packaging/docker/deb-docker/pom.xml | 4 +- .../docker/rpm-docker/SPECS/metron.spec | 6 +- .../packaging/docker/rpm-docker/pom.xml | 4 +- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/.gitignore |49 +- metron-interface/metron-alerts/.nvmrc | 1 + metron-interface/metron-alerts/README.md|16 +- metron-interface/metron-alerts/angular-cli.json |65 - metron-interface/metron-alerts/angular.json | 153 + .../metron-alerts/e2e/tsconfig.json |17 +- metron-interface/metron-alerts/karma.conf.js|18 +- .../metron-alerts/package-lock.json | 24240 - metron-interface/metron-alerts/package.json |67 +- metron-interface/metron-alerts/pom.xml | 8 +- .../metron-alerts/protractor.conf.js|12 +- .../scripts/prepend_license_header.sh | 2 +- .../metron-alerts/scripts/start-dev.sh | 2 +- .../scripts/start-server-for-e2e.sh | 2 +- .../alert-details/alert-details.component.ts| 2 +- .../alerts/alerts-list/alerts-list.component.ts | 6 +- .../table-view/table-view.component.spec.ts |21 +- .../table-view/table-view.component.ts | 2 +- .../alerts-list/tree-view/tree-group-data.ts| 2 +- .../tree-view/tree-view.component.spec.ts | 6 +- .../tree-view/tree-view.component.ts| 2 +- .../configure-table.component.ts| 4 +- .../alerts/meta-alerts/meta-alerts.component.ts | 2 +- .../saved-searches.component.spec.ts| 9 +- .../saved-searches/saved-searches.component.ts | 5 +- .../metron-alerts/src/app/app-routing.module.ts | 2 +- .../metron-alerts/src/app/app.component.spec.ts | 4 +- .../metron-alerts/src/app/app.module.ts | 8 +- .../default-headers.interceptor.ts |34 + .../src/app/login/login.component.spec.ts | 4 +- .../metron-alerts/src/app/model/rest-error.ts | 4 +- .../pcap/pcap-filters/pcap-filters.component.ts | 5 +- .../pcap-panel/pcap-panel.component.spec.ts | 9 +- .../app/pcap/pcap-panel/pcap-panel.component.ts | 4 +- .../metron-alerts/src/app/pcap/pcap.module.ts |14 +- .../src/app/pcap/service/pcap.service.spec.ts | 236 +- .../src/app/pcap/service/pcap.service.ts| 115 +- .../src/app/service/alerts.service.ts
[10/51] [abbrv] metron git commit: METRON-1759 PCAP UI: Removing wrong Input annotations from pcap panel component (tiborm via nickwallen) closes apache/metron#1180
METRON-1759 PCAP UI: Removing wrong Input annotations from pcap panel component
(tiborm via nickwallen) closes apache/metron#1180
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3372aa36
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3372aa36
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3372aa36
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3372aa360315b537ee675ecb7d79fcd6885ba13c
Parents: b84c874
Author: tiborm
Authored: Wed Sep 19 10:39:38 2018 -0400
Committer: nickallen
Committed: Wed Sep 19 10:39:38 2018 -0400
--
.../src/app/pcap/pcap-panel/pcap-panel.component.ts| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3372aa36/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 8e4c375..f767020 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -32,9 +32,9 @@ import { RestError } from '../../model/rest-error';
})
export class PcapPanelComponent implements OnInit, OnDestroy {
- @Input() pdml: Pdml = null;
- @Input() pcapRequest: PcapRequest;
- @Input() resetPaginationForSearch: boolean;
+ pdml: Pdml = null;
+ pcapRequest: PcapRequest;
+ resetPaginationForSearch: boolean;
statusSubscription: Subscription;
cancelSubscription: Subscription;
[41/51] [abbrv] metron git commit: METRON-1805 Provide a default value for the Storm topology.max.spout.pending setting (merrimanr) closes apache/metron#1221
METRON-1805 Provide a default value for the Storm topology.max.spout.pending setting (merrimanr) closes apache/metron#1221 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62141506 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62141506 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62141506 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 621415067dc727838da8e14925c15120b0e479de Parents: 6478966 Author: merrimanr Authored: Mon Oct 15 14:50:10 2018 -0500 Committer: rmerriman Committed: Mon Oct 15 14:50:10 2018 -0500 -- .../METRON/CURRENT/configuration/metron-indexing-env.xml | 4 ++-- .../METRON/CURRENT/configuration/metron-profiler-env.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index 3d268b7..ebc9759 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -146,7 +146,7 @@ ra_indexing_topology_max_spout_pending Indexing Topology Spout Max Pending Tuples - +300 Indexing Max Pending for Random Access true @@ -155,7 +155,7 @@ batch_indexing_topology_max_spout_pending Indexing Topology Spout Max Pending Tuples - +300 Indexing Max Pending for HDFS true http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml index 07b8d11..c45d799 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -165,7 +165,7 @@ profiler_topology_max_spout_pending Profiler Topology Spout Max Pending Tuples Spout Max Pending Tuples - +300 true
[08/51] [abbrv] metron git commit: METRON-1770 Add Docs for Running the Profiler with Spark on YARN (nickwallen) closes apache/metron#1189
METRON-1770 Add Docs for Running the Profiler with Spark on YARN (nickwallen)
closes apache/metron#1189
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f83f0ac0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f83f0ac0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f83f0ac0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: f83f0ac06622e091a09d9f256f817e7235c63e53
Parents: cad2f40
Author: nickwallen
Authored: Wed Sep 19 10:01:50 2018 -0400
Committer: nickallen
Committed: Wed Sep 19 10:01:50 2018 -0400
--
.../metron-profiler-spark/README.md | 94 ++--
.../src/main/config/batch-profiler.properties | 8 +-
2 files changed, 76 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f83f0ac0/metron-analytics/metron-profiler-spark/README.md
--
diff --git a/metron-analytics/metron-profiler-spark/README.md
b/metron-analytics/metron-profiler-spark/README.md
index d137e51..3d7017c 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -22,8 +22,8 @@ This project allows profiles to be executed using [Apache
Spark](https://spark.a
* [Introduction](#introduction)
* [Getting Started](#getting-started)
* [Installation](#installation)
-* [Configuring the Profiler](#configuring-the-profiler)
* [Running the Profiler](#running-the-profiler)
+* [Configuring the Profiler](#configuring-the-profiler)
## Introduction
@@ -129,6 +129,73 @@ The Batch Profiler requires Spark version 2.3.0+.
find ./ -name "metron-profiler-spark*.deb"
```
+## Running the Profiler
+
+A script located at `$METRON_HOME/bin/start_batch_profiler.sh` has been
provided to simplify running the Batch Profiler. This script makes the
following assumptions.
+
+ * The script builds the profiles defined in
`$METRON_HOME/config/zookeeper/profiler.json`.
+
+ * The properties defined in `$METRON_HOME/config/batch-profiler.properties`
are passed to both the Profiler and Spark. You can define both Spark and
Profiler properties in this same file.
+
+ * The script assumes that Spark is installed at
`/usr/hdp/current/spark2-client`. This can be overridden if you define an
environment variable called `SPARK_HOME` prior to executing the script.
+
+### Advanced Usage
+
+The Batch Profiler may also be started using `spark-submit` as follows. See
the Spark Documentation for more information about
[`spark-submit`](https://spark.apache.org/docs/latest/submitting-applications.html#launching-applications-with-spark-submit).
+
+```
+${SPARK_HOME}/bin/spark-submit \
+--class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \
+--properties-file ${SPARK_PROPS_FILE} \
+${METRON_HOME}/lib/metron-profiler-spark-*.jar \
+--config ${PROFILER_PROPS_FILE} \
+--profiles ${PROFILES_FILE}
+```
+
+The Batch Profiler accepts the following arguments when run from the command
line as shown above. All arguments following the Profiler jar are passed to
the Profiler. All argument preceeding the Profiler jar are passed to Spark.
+
+| Argument | Description
+|--- |---
+| -p, --profiles | The path to a file containing the profile definitions.
+| -c, --config | The path to the profiler properties file.
+| -g, --globals| The path to a properties file containing global
properties.
+| -h, --help | Print the help text.
+
+### Spark Execution
+
+Spark supports a number of different [cluster
managers](https://spark.apache.org/docs/latest/cluster-overview.html#cluster-manager-types).
The underlying cluster manager is transparent to the Profiler. To run the
Profiler on a particular cluster manager, it is just a matter of setting the
appropriate options as defined in the Spark documentation.
+
+ Local Mode
+
+By default, the Batch Profiler instructs Spark to run in local mode. This
will run all of the Spark execution components within a single JVM. This mode
is only useful for testing with a limited set of data.
+
+`$METRON_HOME/config/batch-profiler.properties`
+```
+spark.master=local
+```
+
+ Spark on YARN
+
+To run the Profiler using [Spark on
YARN](https://spark.apache.org/docs/latest/running-on-yarn.html#running-spark-on-yarn),
at a minimum edit the value of `spark.master` as shown. In many cases it also
makes sense to set the YARN [deploy
mode](https://spark.apache.org/docs/latest/running-on-yarn.html#launching-spark-on-yarn)
to `cluster`.
+
+`$METRON_HOME/config/batch-profiler.properties`
+```
+spark.master=yarn
+spark.submit.deployMode=cluster
+```
+
+See the Spark documentation for information on how to further control the
execution of
[39/51] [abbrv] metron git commit: METRON-1801 Allow Customization of Elasticsearch Document ID (nickwallen) closes apache/metron#1218
METRON-1801 Allow Customization of Elasticsearch Document ID (nickwallen)
closes apache/metron#1218
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/90c5e1d2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/90c5e1d2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/90c5e1d2
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 90c5e1d2896b8a95bd6c928e3dab96f1fd85167c
Parents: 9b67100
Author: nickwallen
Authored: Thu Oct 11 16:29:40 2018 -0400
Committer: nickallen
Committed: Thu Oct 11 16:29:40 2018 -0400
--
Upgrading.md| 23 ++-
.../METRON/CURRENT/configuration/metron-env.xml | 9 ++
.../CURRENT/package/scripts/metron_service.py | 5 +
.../package/scripts/params/params_linux.py | 1 +
.../METRON/CURRENT/themes/metron_theme.json | 10 ++
metron-platform/metron-elasticsearch/README.md | 37 -
metron-platform/metron-elasticsearch/pom.xml| 17 ---
.../dao/ElasticsearchRetrieveLatestDao.java | 147 ---
.../dao/ElasticsearchSearchDao.java | 28 +++-
.../elasticsearch/utils/ElasticsearchUtils.java | 100 +
.../writer/ElasticsearchWriter.java | 39 +++--
.../elasticsearch/dao/ElasticsearchDaoTest.java | 10 +-
.../ElasticsearchSearchIntegrationTest.java | 20 ++-
.../indexing/dao/SearchIntegrationTest.java | 21 +--
.../integration/SolrSearchIntegrationTest.java | 19 +++
15 files changed, 341 insertions(+), 145 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/Upgrading.md
--
diff --git a/Upgrading.md b/Upgrading.md
index 2124ac5..c3b2a0f 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -16,9 +16,22 @@ See the License for the specific language governing
permissions and
limitations under the License.
-->
# Upgrading
+
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.6.0 to 0.6.1
+
+### [METRON-1801 Allow Customization of Elasticsearch Document
ID](https://issues.apache.org/jira/browse/METRON-1801)
+
+A global property named `es.document.id` was added to define the field from
which the document ID is set when a message is indexed by Elasticsearch. To
allow Elasticsearch to define its own document id, this property should be set
to a blank or empty string. The client will not set the document ID and
Elasticsearch will define its own. In most cases allowing Elasticsearch to
define the document ID is the most performant option. This is now the default
behavior.
+
+Metron versions 0.6.0 and earlier defined the document ID using the Metron
GUID, which is a randomized UUID using Java's `UUID.randomUUID()`. Using a
randomized UUID can negatively impact Elasticsearch indexing performance. To
maintain backwards compatibility with legacy versions of Metron use the
following global property setting.
+
+```
+es.document.id = guid
+```
+
## 0.4.2 to 0.5.0
### [METRON-941: native PaloAlto parser corrupts message when having a comma
in the payload](https://issues.apache.org/jira/browse/METRON-941)
@@ -89,7 +102,7 @@ For a more detailed description, please see
metron-platform/metron-elasticsearch
### Description
-In the 0.4.2 release,
+In the 0.4.2 release,
## 0.3.1 to 0.4.0
@@ -107,7 +120,7 @@ This effectively limits the build environment to Docker
supported [platforms](ht
Description
As of 0.3.0 the indexing configuration
-* Is held in the enrichment configuration for a sensor
+* Is held in the enrichment configuration for a sensor
* Has properties which control every writers (i.e. HDFS, solr or
elasticsearch).
In the 0.3.1 release, this configuration has been broken out
@@ -136,7 +149,7 @@ You would create a file to configure each writer for sensor
`foo` called `$METRO
"batchSize" : 100,
"enabled" : true
},
- "hdfs" : {
+ "hdfs" : {
"index" : "foo",
"batchSize" : 100,
"enabled" : true
@@ -151,7 +164,7 @@ You would create a file to configure each writer for sensor
`foo` called `$METRO
As of 0.3.0, threat triage rules were defined as a simple Map associating a
Stellar expression with a score.
As of 0.3.1, due to the fact that there may be many threat triage rules, we
have made the rules more complex.
To help organize these, we have made the threat triage objects in their own
right that contain optional name and optional comment fields.
-
+
This essentially makes the risk level rules slightly more complex. The format
goes from:
```
"riskLevelRules" : {
@@ -169,7 +182,7 @@ to:
}
]
```
-
+
Migration
For every sensor enrichment configuration, you will need to migrate the
[05/51] [abbrv] metron git commit: METRON-1777: Fix Elasticsearch X-Pack sample pom in documentation (mmiklavc via mmiklavc) closes apache/metron#1196
METRON-1777: Fix Elasticsearch X-Pack sample pom in documentation (mmiklavc via
mmiklavc) closes apache/metron#1196
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8b2f297f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8b2f297f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8b2f297f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 8b2f297fb58df1c2f01fa34a9d2ed5ccd183a7ce
Parents: 02a1280
Author: mmiklavc
Authored: Tue Sep 18 08:55:45 2018 -0600
Committer: Michael Miklavcic
Committed: Tue Sep 18 08:55:45 2018 -0600
--
metron-deployment/Kerberos-manual-setup.md | 13 +
1 file changed, 5 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8b2f297f/metron-deployment/Kerberos-manual-setup.md
--
diff --git a/metron-deployment/Kerberos-manual-setup.md
b/metron-deployment/Kerberos-manual-setup.md
index d34a62a..4bd4516 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -30,7 +30,7 @@ This document provides instructions for kerberizing Metron's
Vagrant-based devel
* [Start Metron](#start-metron)
* [Push Data](#push-data)
* [More Information](#more-information)
-* [Elasticseach X-Pack](#x-pack)
+* [Elasticseach X-Pack](#X-Pack)
Setup
-
@@ -568,7 +568,7 @@ X-Pack
Add the `es.client.settings` to global.json
```
-/usr/metron/0.6.0/config/zookeeper/global.json ->
+$METRON_HOME/config/zookeeper/global.json ->
"es.client.settings" : {
"es.client.class" :
"org.elasticsearch.xpack.client.PreBuiltXPackTransportClient",
@@ -650,10 +650,6 @@ X-Pack
log4j
log4j
-
-org.apache.logging.log4j
-log4j-api
-
@@ -662,7 +658,7 @@ X-Pack
org.apache.maven.plugins
maven-shade-plugin
-2.4.3
+3.2.0
true
@@ -728,7 +724,8 @@ X-Pack
1. Once you've built the `elasticsearch-xpack-shaded-5.6.2.jar`, it needs to
be made available to Storm when you submit the topology. Create a contrib
directory for indexing and put the jar file in this directory.
```
-/usr/metron/0.6.0/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar
+mkdir $METRON_HOME/indexing_contrib
+cp elasticsearch-xpack-shaded-5.6.2.jar
$METRON_HOME/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar
```
1. Now you can restart the Elasticsearch topology. Note, you should perform
this step manually, as follows.
[09/51] [abbrv] metron git commit: METRON-1772 Support alternative input formats in the Batch Profiler (nickwallen) closes apache/metron#1191
METRON-1772 Support alternative input formats in the Batch Profiler (nickwallen) closes apache/metron#1191 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1545978e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1545978e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1545978e Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 1545978e169a01e4a06735b8713c8fa65373a394 Parents: f83f0ac Author: nickwallen Authored: Wed Sep 19 10:11:28 2018 -0400 Committer: nickallen Committed: Wed Sep 19 10:11:28 2018 -0400 -- .../metron-profiler-spark/README.md | 47 +- metron-analytics/metron-profiler-spark/pom.xml | 18 ++-- .../metron/profiler/spark/BatchProfiler.java| 21 +++-- .../profiler/spark/cli/BatchProfilerCLI.java| 40 +++-- .../spark/cli/BatchProfilerCLIOptions.java | 10 ++- .../spark/BatchProfilerIntegrationTest.java | 91 +--- 6 files changed, 189 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/README.md -- diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md index 3d7017c..99e8c7e 100644 --- a/metron-analytics/metron-profiler-spark/README.md +++ b/metron-analytics/metron-profiler-spark/README.md @@ -131,6 +131,14 @@ The Batch Profiler requires Spark version 2.3.0+. ## Running the Profiler +* [Usage](#usage) +* [Advanced Usage](#advanced-usage) +* [Spark Execution](#spark-execution) +* [Kerberos](#kerberos) +* [Input Formats](#input-formats) + +### Usage + A script located at `$METRON_HOME/bin/start_batch_profiler.sh` has been provided to simplify running the Batch Profiler. This script makes the following assumptions. * The script builds the profiles defined in `$METRON_HOME/config/zookeeper/profiler.json`. @@ -156,11 +164,28 @@ The Batch Profiler accepts the following arguments when run from the command lin | Argument | Description |--- |--- -| -p, --profiles | The path to a file containing the profile definitions. -| -c, --config | The path to the profiler properties file. -| -g, --globals| The path to a properties file containing global properties. +| -p, --profiles | Path to the profile definitions. +| -c, --config | Path to the profiler properties file. +| -g, --globals| Path to the Stellar global config file. +| -r, --reader | Path to properties for the DataFrameReader. | -h, --help | Print the help text. + `--profiles` + +The path to a file containing the profile definition in JSON. + + `--config` + +The path to a file containing key-value properties for the Profiler. This file would contain the properties described under [Configuring the Profiler](#configuring-the-profiler). + + `--globals` + +The path to a file containing key-value properties that define the global properties. This can be used to customize how certain Stellar functions behave during execution. + + `--reader` + +The path to a file containing key-value properties that are passed to the DataFrameReader when reading the input telemetry. This allows additional customization for how the input telemetry is read. + ### Spark Execution Spark supports a number of different [cluster managers](https://spark.apache.org/docs/latest/cluster-overview.html#cluster-manager-types). The underlying cluster manager is transparent to the Profiler. To run the Profiler on a particular cluster manager, it is just a matter of setting the appropriate options as defined in the Spark documentation. @@ -191,10 +216,24 @@ The following command can be useful to review the logs generated when the Profil yarn logs -applicationId ``` - Kerberos +### Kerberos See the Spark documentation for information on running the Batch Profiler in a [secure, kerberized cluster](https://spark.apache.org/docs/latest/running-on-yarn.html#running-in-a-secure-cluster). +### Input Formats + +The Profiler can consume archived telemetry stored in a variety of input formats. By default, it is configured to consume the text/json that Metron archives in HDFS. This is often not the best format for archiving telemetry. If you choose a different format, you should be able to configure the Profiler to consume it by doing the following. + +1. Edit [`profiler.batch.input.format`](#profilerbatchinputformat) and [`profiler.batch.input.path`](#profilerbatchinputpath) as needed. For example, to read ORC you might do the following. + + `$METRON_HOME/config/batch-profiler.properties` + ``` +
[02/51] [abbrv] metron git commit: Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
--
diff --cc
metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
index 000,000..4a324cf
new file mode 100644
--- /dev/null
+++ b/metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
@@@ -1,0 -1,0 +1,100 @@@
++{"adapter.threatinteladapter.end.ts":"1530978697769","qclass_name":"qclass-32769","bro_timestamp":"1530978687.836793","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696551","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696550","adapter.hostfromjsonlistadapter.end.ts":"1530978696606","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696857","uid":"CGs8rS1rqhyXRRgA64","protocol":"dns","original_string":"DNS
| AA:false qclass_name:qclass-32769 id.orig_p:5353 qtype_name:PTR qtype:12
rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false
RA:false uid:CGs8rS1rqhyXRRgA64 RD:false proto:udp id.orig_h:192.168.66.1 Z:0
qclass:32769 ts:1530978687.836793
id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697808","enrichmentjoinbolt.joiner.ts":"1530978696932","adapter.hostfromjsonlistadapter.begin.ts":"1530978696606","threatintelsplitterbolt.splitter.begin.ts
":"1530978696949","Z":0,"ip_src_addr":"192.168.66.1","qclass":32769,"timestamp":1530978687836,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696857","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697764","ip_src_port":5353,"proto":"udp","guid":"90751ce5-703d-4b9f-8c2d-8e5c42e72262"}
++{"adapter.threatinteladapter.end.ts":"1530978697772","bro_timestamp":"1530978687.77394","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696605","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696605","adapter.hostfromjsonlistadapter.end.ts":"1530978696649","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696857","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CBJatv2DcsW8fow3Dg","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
| id.orig_p:49186 status_code:200 method:GET request_body_len:0 id.resp_p:80
uri:\/ tags:[] uid:CBJatv2DcsW8fow3Dg
referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745
resp_mime_types:[\"text\\\/html\"] trans_depth:1 ho
st:r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in
status_msg:OK id.orig_h:192.168.138.158 response_body_len:121635
user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64;
Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR
3.0.30729; Media Center PC 6.0) ts:1530978687.77394 id.resp_h:62.75.195.236
resp_fuids:[\"F77a061yn9H0cUBGVa\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697808","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978696943","adapter.hostfromjsonlistadapter.begin.ts":"1530978696607","threatintelsplitterbolt.splitter.begin.ts":"1530978696952","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0
(compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR
2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC
6.0)","resp_fuids":[
"F77a061yn9H0cUBGVa"],"timestamp":1530978687773,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696858","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697769","ip_src_port":49186,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"f5b315b0-e776-481a-9f28-765fdb19e6e8","response_body_len":121635}
++{"adapter.threatinteladapter.end.ts":"1530978697776","bro_timestamp":"1530978687.916811","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978696606","enrichmentsplitterbolt.splitter.begin.ts":"1530978696606","adapter.hostfromjsonlistadapter.end.ts":"1530978696650","adapter.geoadapter.begin.ts":"1530978696858","uid":"CUrRne3iLIxXavQtci","trans_depth":6,"protocol":"http","original_string":"HTTP
| id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080
[35/51] [abbrv] metron git commit: METRON-1813 Stellar REPL Not Initialized with Client JAAS (nickwallen) closes apache/metron#1232
METRON-1813 Stellar REPL Not Initialized with Client JAAS (nickwallen) closes
apache/metron#1232
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/060d17e9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/060d17e9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/060d17e9
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 060d17e992a73bc5b65bb9a52d044a3f672ba2c1
Parents: 1f1b9cd
Author: nickwallen
Authored: Wed Oct 10 09:54:21 2018 -0400
Committer: nickallen
Committed: Wed Oct 10 09:54:21 2018 -0400
--
metron-platform/metron-common/src/main/scripts/stellar | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/060d17e9/metron-platform/metron-common/src/main/scripts/stellar
--
diff --git a/metron-platform/metron-common/src/main/scripts/stellar
b/metron-platform/metron-common/src/main/scripts/stellar
index c831c62..34097f2 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -28,10 +28,15 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ];
then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
+export METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ source $METRON_SYSCONFIG
+fi
+
+export METRON_VERSION="${METRON_VERSION:-${project.version}}"
+export METRON_HOME="${METRON_HOME:-/usr/metron/$METRON_VERSION}"
export HBASE_CONFIGS=$(hbase classpath)
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar)
-java $JVMFLAGS -cp
"${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS"
org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
+java $METRON_JVMFLAGS -cp
"${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS"
org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
[06/51] [abbrv] metron git commit: METRON-1760 Kill PCAP job should prompt for confirmation (ruffle via nickwallen) closes apache/metron#1199
METRON-1760 Kill PCAP job should prompt for confirmation (ruffle via
nickwallen) closes apache/metron#1199
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/91c410fd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/91c410fd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/91c410fd
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 91c410fd2016d8776aba650b41239c971b706d01
Parents: 8b2f297
Author: ruffle
Authored: Tue Sep 18 11:11:44 2018 -0400
Committer: nickallen
Committed: Tue Sep 18 11:11:44 2018 -0400
--
.../metron-alerts/package-lock.json | 14 +++
metron-interface/metron-alerts/package.json | 1 +
.../pcap/pcap-panel/pcap-panel.component.html | 17 +++-
.../pcap-panel/pcap-panel.component.spec.ts | 36 +++-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 1 +
.../metron-alerts/src/app/pcap/pcap.module.ts | 4 +-
.../metron-alerts/src/confirm-popover.scss | 95
metron-interface/metron-alerts/src/styles.scss | 1 +
8 files changed, 166 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/package-lock.json
--
diff --git a/metron-interface/metron-alerts/package-lock.json
b/metron-interface/metron-alerts/package-lock.json
index 7d25b51..f927ffe 100644
--- a/metron-interface/metron-alerts/package-lock.json
+++ b/metron-interface/metron-alerts/package-lock.json
@@ -5808,6 +5808,15 @@
"integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=",
"dev": true
},
+"angular-confirmation-popover": {
+ "version": "4.2.0",
+ "resolved":
"https://registry.npmjs.org/angular-confirmation-popover/-/angular-confirmation-popover-4.2.0.tgz;,
+ "integrity":
"sha512-ItCPzV52user93NRk9rF4Rp8NpawBWJdkNf8+6lH//f5i/N5HY0Aq5Hcch3xk19h9P48k0WZnfwOQL181xe4MQ==",
+ "requires": {
+"positioning": "^1.3.1",
+"tslib": "^1.9.0"
+ }
+},
"ansi-html": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz;,
@@ -14148,6 +14157,11 @@
}
}
},
+"positioning": {
+ "version": "1.4.0",
+ "resolved":
"https://registry.npmjs.org/positioning/-/positioning-1.4.0.tgz;,
+ "integrity":
"sha512-LbN+mgAXtcDdN46xMJ3yZwjndqqYJODaO5qKmU+MVMu5tL3K2dlm1Qha/zh1k2JAFym5HDaZpnPfO4gr91VTRw=="
+},
"posix-character-classes": {
"version": "0.1.1",
"resolved":
"https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz;,
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/package.json
--
diff --git a/metron-interface/metron-alerts/package.json
b/metron-interface/metron-alerts/package.json
index 2133b74..14172fd 100644
--- a/metron-interface/metron-alerts/package.json
+++ b/metron-interface/metron-alerts/package.json
@@ -26,6 +26,7 @@
"@types/jquery": "^3.3.4",
"ace-builds": "^1.2.6",
"ajv": "^6.5.1",
+"angular-confirmation-popover": "^4.2.0",
"bootstrap": "4.0.0-alpha.6",
"core-js": "^2.4.1",
"font-awesome": "^4.7.0",
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 6b18a5f..37e21ee 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -19,7 +19,22 @@
{{progressWidth}}%
-
+
+
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index aa9b799..29bd649 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -27,6 +27,7 @@ import { By } from
'../../../../node_modules/@angular/platform-browser';
import { PcapRequest } from '../model/pcap.request';
import { of, defer } from
[50/51] [abbrv] metron git commit: METRON-1826 Update librdkafka and devtoolset (JonZeolla via jonzeolla) closes apache/metron#1238
METRON-1826 Update librdkafka and devtoolset (JonZeolla via jonzeolla) closes
apache/metron#1238
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ba18026
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ba18026
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ba18026
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0ba18026222f004316b96633780509c71934121e
Parents: 739782a
Author: JonZeolla
Authored: Mon Oct 22 19:48:56 2018 -0400
Committer: jonzeolla
Committed: Mon Oct 22 19:48:56 2018 -0400
--
.../ansible/roles/bro/tasks/bro.yml | 4 +--
.../ansible/roles/bro/tasks/dependencies.yml| 4 +--
.../roles/bro/tasks/metron-bro-plugin-kafka.yml | 4 +--
.../ansible/roles/librdkafka/defaults/main.yml | 4 +--
metron-deployment/development/centos6/README.md | 26 ++--
.../development/ubuntu14/README.md | 26 ++--
6 files changed, 56 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/bro.yml
--
diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml
b/metron-deployment/ansible/roles/bro/tasks/bro.yml
index 222ef0e..9ba3ffa 100644
--- a/metron-deployment/ansible/roles/bro/tasks/bro.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml
@@ -30,8 +30,8 @@
- name: Compile and Install bro
shell: "{{ item }}"
environment:
-CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
args:
chdir: "/tmp/bro-{{ bro_version }}"
creates: "{{ bro_home }}/bin/bro"
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
--
diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
index 3cd3bae..f8b1e28 100644
--- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
@@ -38,8 +38,8 @@
- perl
- crontabs
- net-tools
-- devtoolset-4-gcc
-- devtoolset-4-gcc-c++
+- devtoolset-6-gcc
+- devtoolset-6-gcc-c++
- python27
- rh-git29
register: result
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
--
diff --git
a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
index 7043387..ec7af97 100644
--- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -30,8 +30,8 @@
environment:
PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
-CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka
--force --version {{ metron_bro_plugin_kafka_version }}"
- name: Configure bro-kafka plugin
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
--
diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
index 063c22f..6e25cb3 100644
--- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
+++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
@@ -15,6 +15,6 @@
# limitations under the License.
#
---
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+librdkafka_version: 0.11.5
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz
librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/development/centos6/README.md
--
diff --git a/metron-deployment/development/centos6/README.md
b/metron-deployment/development/centos6/README.md
index e873452..99ec967 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@
[40/51] [abbrv] metron git commit: METRON-1821 Align prepare-release-candidate with documentation (JonZeolla via jonzeolla) closes apache/metron#1236
METRON-1821 Align prepare-release-candidate with documentation (JonZeolla via
jonzeolla) closes apache/metron#1236
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/64789660
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/64789660
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/64789660
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 647896604f938ca195bf25885651f9be3e897c71
Parents: 90c5e1d
Author: JonZeolla
Authored: Mon Oct 15 09:42:44 2018 -0400
Committer: jonzeolla
Committed: Mon Oct 15 09:42:44 2018 -0400
--
.../release-utils/prepare-release-candidate | 23 +---
1 file changed, 10 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/64789660/dev-utilities/release-utils/prepare-release-candidate
--
diff --git a/dev-utilities/release-utils/prepare-release-candidate
b/dev-utilities/release-utils/prepare-release-candidate
index bedee9b..635285c 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -33,9 +33,10 @@
BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_N
DEV_REPO="https://dist.apache.org/repos/dist/dev/metron;
RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron;
-PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
+PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git;
RC_PREFIX=rc
+UPPER_RC_PREFIX="$(tr '[:lower:]' '[:upper:]' <<< ${RC_PREFIX})"
TAG_POSTFIX="-release"
CONFIG_FILE=~/.metron-prepare-release-candidate
@@ -223,33 +224,29 @@ if [ "${PRACTICE_RUN}" = true ]; then
printf "This is a practice run. Not running \n" "${CAPITAL_REPO}" "$VERSION"
else
printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
- git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}"
+ git push --set-upstream origin "${CAPITAL_REPO}_${VERSION}"
fi
# Create directory for release artifacts
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
- ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}"
+ ART_DIR="$WORKDIR/dev/metron/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
else
# We're using a sub module, so put it in it's own directory.
- ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}"
+
ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
fi
mkdir -p "$ART_DIR"
# Setup various parameters we need for the release artifacts
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
- CORE_PREFIX="apache-metron-"
+ CORE_PREFIX="apache-metron_"
ARTIFACT_PREFIX="${CORE_PREFIX}"
TAG_VERSION="${CURRENT_VERSION}"
TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
- PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
- ARTIFACT_PREFIX="${PLUGIN_PREFIX}"
+ BRO_PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
+ ARTIFACT_PREFIX="${BRO_PLUGIN_PREFIX}"
TAG_VERSION="${CURRENT_VERSION}"
- TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
- # Handle special tag case from prior release
- if [ "${TAG_VERSION}" = "0.1" ]; then
-TAG="0.1"
- fi
+ TAG="${BRO_PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
else
# If we ever add new modules, add them as needed.
printf "Unrecognized module: %s\n" "${CHOSEN_REPO}"
@@ -324,7 +321,7 @@ if [ "${PRACTICE_RUN}" = true ]; then
printf "\n" ${COMMIT_DIR}
printf "\n" "${CHOSEN_REPO}"
"${COMMIT_DIR}"
else
- printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n"
+ printf "Adding artifacts for Metron ${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}
to dev SVN\n"
# Metron goes in the root of the dir, submodules go in folder
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
cd "$WORKDIR/dev/metron/"
[23/51] [abbrv] metron git commit: METRON-1791 Add GUID to Messages Produced by Profiler (nickwallen) closes apache/metron#1210
METRON-1791 Add GUID to Messages Produced by Profiler (nickwallen) closes
apache/metron#1210
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7e222fa4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7e222fa4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7e222fa4
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 7e222fa47aedc4ab3bafec54590ec0bc73b5f75c
Parents: b872fdc
Author: nickwallen
Authored: Thu Oct 4 17:45:32 2018 -0400
Committer: nickallen
Committed: Thu Oct 4 17:45:32 2018 -0400
--
.../metron/profiler/storm/KafkaEmitter.java | 31 +---
.../zookeeper/triage-result/profiler.json | 20
.../metron/profiler/storm/KafkaEmitterTest.java | 4 ++
.../integration/ProfilerIntegrationTest.java| 52 ++--
4 files changed, 85 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
--
diff --git
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
index af1fbca..adbde1b 100644
---
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
+++
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
@@ -21,6 +21,7 @@ package org.apache.metron.profiler.storm;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.ClassUtils;
+import org.apache.metron.common.Constants;
import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -33,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
import java.util.Map;
+import java.util.UUID;
/**
* Responsible for emitting a {@link ProfileMeasurement} to an output stream
that will
@@ -42,6 +44,14 @@ public class KafkaEmitter implements
ProfileMeasurementEmitter, Serializable {
protected static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PROFILE_FIELD = "profile";
+ public static final String ENTITY_FIELD = "entity";
+ public static final String PERIOD_ID_FIELD = "period";
+ public static final String PERIOD_START_FIELD = "period.start";
+ public static final String PERIOD_END_FIELD = "period.end";
+ public static final String TIMESTAMP_FIELD = "timestamp";
+ public static final String ALERT_FIELD = "is_alert";
+
/**
* The stream identifier used for this destination;
*/
@@ -126,14 +136,15 @@ public class KafkaEmitter implements
ProfileMeasurementEmitter, Serializable {
private JSONObject createMessage(ProfileMeasurement measurement) {
JSONObject message = new JSONObject();
-message.put("profile", measurement.getDefinition().getProfile());
-message.put("entity", measurement.getEntity());
-message.put("period", measurement.getPeriod().getPeriod());
-message.put("period.start", measurement.getPeriod().getStartTimeMillis());
-message.put("period.end", measurement.getPeriod().getEndTimeMillis());
-message.put("timestamp", System.currentTimeMillis());
-message.put("source.type", sourceType);
-message.put("is_alert", "true");
+message.put(PROFILE_FIELD, measurement.getDefinition().getProfile());
+message.put(ENTITY_FIELD, measurement.getEntity());
+message.put(PERIOD_ID_FIELD, measurement.getPeriod().getPeriod());
+message.put(PERIOD_START_FIELD,
measurement.getPeriod().getStartTimeMillis());
+message.put(PERIOD_END_FIELD, measurement.getPeriod().getEndTimeMillis());
+message.put(TIMESTAMP_FIELD, System.currentTimeMillis());
+message.put(Constants.SENSOR_TYPE, sourceType);
+message.put(ALERT_FIELD, "true");
+message.put(Constants.GUID, UUID.randomUUID().toString());
return message;
}
@@ -158,6 +169,10 @@ public class KafkaEmitter implements
ProfileMeasurementEmitter, Serializable {
this.streamId = streamId;
}
+ public String getSourceType() {
+return sourceType;
+ }
+
public void setSourceType(String sourceType) {
this.sourceType = sourceType;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/triage-result/profiler.json
--
diff --git
