metron git commit: METRON-1104 Build Failure - Due to Bootstrap version change (iraghumitra via justinleet) closes apache/metron#691

2017-08-11 Thread leet 
Repository: metron
Updated Branches:
  refs/heads/master 82bf89ba4 -> 86b0f137e


METRON-1104 Build Failure - Due to Bootstrap version change (iraghumitra via 
justinleet) closes apache/metron#691


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/86b0f137
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/86b0f137
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/86b0f137

Branch: refs/heads/master
Commit: 86b0f137e29a78a5ec849a8dbf0cb8a2e5bb3dbe
Parents: 82bf89b
Author: iraghumitra 
Authored: Fri Aug 11 13:35:33 2017 -0400
Committer: leet 
Committed: Fri Aug 11 13:35:33 2017 -0400

--
 metron-interface/metron-alerts/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/86b0f137/metron-interface/metron-alerts/package.json
--
diff --git a/metron-interface/metron-alerts/package.json 
b/metron-interface/metron-alerts/package.json
index dd249cc..446c40d 100644
--- a/metron-interface/metron-alerts/package.json
+++ b/metron-interface/metron-alerts/package.json
@@ -1,6 +1,6 @@
 {
   "name": "metron-alerts",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "license": "Apache-2.0",
   "angular-cli": {},
   "scripts": {
@@ -22,7 +22,7 @@
 "@angular/platform-browser-dynamic": "^4.0.0",
 "@angular/router": "^4.0.0",
 "ace-builds": "^1.2.6",
-"bootstrap": "^4.0.0-alpha.6",
+"bootstrap": "4.0.0-alpha.6",
 "core-js": "^2.4.1",
 "font-awesome": "^4.7.0",
 "rxjs": "^5.1.0",

metron git commit: METRON-1094: MaaS will not start due to classpath error relating to Yarn closes apache/incubator-metron#688

2017-08-11 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 3d95fdf90 -> 82bf89ba4


METRON-1094: MaaS will not start due to classpath error relating to Yarn closes 
apache/incubator-metron#688


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82bf89ba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82bf89ba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82bf89ba

Branch: refs/heads/master
Commit: 82bf89ba4ada583ac68dbb44e7fd658ad45e8337
Parents: 3d95fdf
Author: cstella 
Authored: Fri Aug 11 13:30:04 2017 -0400
Committer: cstella 
Committed: Fri Aug 11 13:30:04 2017 -0400

--
 metron-analytics/metron-maas-service/pom.xml | 19 ---
 .../metron/maas/functions/MaaSFunctions.java |  2 +-
 .../apache/metron/maas/service/MockDGAModel.java |  2 +-
 3 files changed, 18 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/82bf89ba/metron-analytics/metron-maas-service/pom.xml
--
diff --git a/metron-analytics/metron-maas-service/pom.xml 
b/metron-analytics/metron-maas-service/pom.xml
index e76aec2..0ac9bac 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -33,19 +33,31 @@
   
   
 
+  org.apache.hadoop
+  hadoop-yarn-api
+  ${hadoop.version}
+  provided
+
+
   org.apache.metron
   metron-maas-common
   ${project.parent.version}
 
 
-  org.apache.metron
-  metron-common
-  ${project.parent.version}
+  de.javakaffee
+  kryo-serializers
+  ${global_kryo_serializers_version}
 
 
   org.apache.metron
   stellar-common
   ${project.parent.version}
+  
+
+  org.apache.hadoop
+  hadoop-auth
+
+  
 
 
   org.apache.hadoop
@@ -59,6 +71,7 @@
   ${hadoop.version}
   provided
 
+
 
   com.sun.jersey.contribs
   jersey-guice

http://git-wip-us.apache.org/repos/asf/metron/blob/82bf89ba/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
--
diff --git 
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
 
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
index e205518..eacb64d 100644
--- 
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
+++ 
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
@@ -29,13 +29,13 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.maas.config.Endpoint;
 import org.apache.metron.maas.config.MaaSConfig;
 import org.apache.metron.maas.config.ModelEndpoint;
 import org.apache.metron.maas.discovery.ServiceDiscoverer;
 import org.apache.metron.maas.util.ConfigUtil;
 import org.apache.metron.maas.util.RESTUtil;
+import org.apache.metron.stellar.common.utils.JSONUtils;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.Stellar;

http://git-wip-us.apache.org/repos/asf/metron/blob/82bf89ba/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
--
diff --git 
a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
 
b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
index 18604b4..66948aa 100644
--- 
a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
+++ 
b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.ImmutableMap;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
-import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.stellar.common.utils.JSONUtils;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.Application;

metron git commit: METRON-1102: Add support for ingesting cybox URI observables from taxii feeds closes apache/incubator-metron#689

2017-08-11 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master b3148a182 -> 3d95fdf90


METRON-1102: Add support for ingesting cybox URI observables from taxii feeds 
closes apache/incubator-metron#689


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d95fdf9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d95fdf9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d95fdf9

Branch: refs/heads/master
Commit: 3d95fdf906993c8d65b76a7e1311c7fb295e19b1
Parents: b3148a1
Author: cstella 
Authored: Fri Aug 11 13:29:07 2017 -0400
Committer: cstella 
Committed: Fri Aug 11 13:29:07 2017 -0400

--
 .../metron-data-management/README.md|  6 ++
 .../dataloads/extractor/stix/StixExtractor.java | 15 +++
 .../stix/types/ObjectTypeHandlers.java  |  1 +
 .../extractor/stix/types/URIHandler.java| 68 ++
 .../extractor/stix/URIHandlerTest.java  | 97 
 5 files changed, 187 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/README.md
--
diff --git a/metron-platform/metron-data-management/README.md 
b/metron-platform/metron-data-management/README.md
index 014f3ef..c351f48 100644
--- a/metron-platform/metron-data-management/README.md
+++ b/metron-platform/metron-data-management/README.md
@@ -97,6 +97,11 @@ Consider the following config for importing STIX documents.  
This is a threat in
 format, so it is particularly relevant and attractive data to import for our 
purposes.  Because STIX is
 a standard format, there is no need to specify the schema or how to interpret 
the documents.
 
+We support the versions of Stix and Cybox supported by
+[java-stix](https://github.com/STIXProject/java-stix/tree/v1.2.0.2):
+* Stix - 
[1.2](https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd)
 and earlier
+* Cybox - 
[2.1](https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd)
 and earlier
+
 We support a subset of STIX messages for importation:
 
 | STIX Type | Specific Type | Enrichment Type Name |
@@ -107,6 +112,7 @@ We support a subset of STIX messages for importation:
 | Address   | MAC   | address:MAC  |
 | Domain| FQDN  | domain:FQDN  |
 | Hostname  |   | hostname |
+| URI   |   | uriobjecttype|
 
 
 NOTE: The enrichment type will be used as the type above.

http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
--
diff --git 
a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
 
b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
index 41b428e..f96bd2d 100644
--- 
a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
+++ 
b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
@@ -30,14 +30,18 @@ import org.mitre.cybox.cybox_2.Observables;
 import org.mitre.stix.common_1.IndicatorBaseType;
 import org.mitre.stix.indicator_2.Indicator;
 import org.mitre.stix.stix_1.STIXPackage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 public class StixExtractor implements Extractor {
+private static final Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 Map config;
 @Override
 public Iterable extract(String line) throws IOException {
@@ -50,11 +54,22 @@ public class StixExtractor implements Extractor {
 if(props != null) {
 ObjectTypeHandler handler = 
ObjectTypeHandlers.getHandlerByInstance(props);
 if (handler != null) {
+if(LOG.isDebugEnabled()) {
+LOG.debug("Found {} for properties {}"
+, handler.getTypeClass().getCanonicalName()
+, props.toXMLString());
+}
 Iterable extractions = 
handler.extract(props, config);
 for(LookupKV extraction : extractions) {

metron git commit: METRON-838 Incorrect set of ts in FireEye parser (bjigmp via justinleet) closes apache/metron#528

2017-08-11 Thread leet 
Repository: metron
Updated Branches:
  refs/heads/master f072ed231 -> b3148a182


METRON-838 Incorrect set of ts in FireEye parser (bjigmp via justinleet) closes 
apache/metron#528


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b3148a18
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b3148a18
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b3148a18

Branch: refs/heads/master
Commit: b3148a18280a4fa50020f4a757dc526c404e0df5
Parents: f072ed2
Author: bjigmp 
Authored: Fri Aug 11 08:40:11 2017 -0400
Committer: leet 
Committed: Fri Aug 11 08:40:11 2017 -0400

--
 .../metron/parsers/fireeye/BasicFireEyeParser.java  |  8 +++-
 .../parsers/fireeye/BasicFireEyeParserTest.java | 16 
 2 files changed, 19 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/b3148a18/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index 04e1591..489eb00 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -122,14 +122,12 @@ public class BasicFireEyeParser extends BasicParser {
month = tsMatcher.group(1);
day = tsMatcher.group(2);
time = tsMatcher.group(3);
-   
-   } else {
-   LOG.warn("Unable to find timestamp in message: {}", 
toParse);
ts = ParserUtils.convertToEpoch(month, day, time, true);
+   } else {
+   LOG.warn("Unable to find timestamp in message: {}", 
toParse);
}
 
-   return ts;
-   
+   return ts;
}
 
private JSONObject parseMessage(String toParse) {

http://git-wip-us.apache.org/repos/asf/metron/blob/b3148a18/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
index 69a6dbd..7a5d2e6 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
@@ -19,6 +19,10 @@ package org.apache.metron.parsers.fireeye;
 
 import java.util.Map;
 import java.util.Map.Entry;
+import java.time.Year;
+import java.time.ZonedDateTime;
+import java.time.ZoneOffset;
+
 import org.apache.metron.parsers.AbstractParserConfigTest;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -57,4 +61,16 @@ public class BasicFireEyeParserTest extends 
AbstractParserConfigTest {
   }
 }
   }
+
+  private final static String fireeyeMessage = "<164>Mar 19 05:24:39 
10.220.15.15 fenotify-851983.alert: 
CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:28:26 UTC 
dvc=10.201.78.57 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org 
dvchost=DEVFEYE1 spt=54527 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 
cn1Label=vlan cn1=0 externalId=851983 cs4Label=link 
cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851983 
dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS";
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testTimestampParsing() throws ParseException {
+JSONObject parsed = parser.parse(fireeyeMessage.getBytes()).get(0);
+JSONParser parser = new JSONParser();
+Map json = (Map) parser.parse(parsed.toJSONString());
+long expectedTimestamp = 
ZonedDateTime.of(Year.now(ZoneOffset.UTC).getValue(), 3, 19, 5, 24, 39, 0, 
ZoneOffset.UTC).toInstant().toEpochMilli();
+Assert.assertEquals(expectedTimestamp, json.get("timestamp"));
+  }
 }