http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml new file mode 100644 index 0000000..616bb24 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa-extension</artifactId> + <name>metron-parser-asa-extension</name> + <version>0.4.1</version> + <packaging>pom</packaging> + + + <description>ASA Parser Extension for Metron</description> + <modules> + <module>metron-parser-asa</module> + <module>metron-parser-asa-bundle</module> + <module>metron-parser-asa-assembly</module> + </modules> +</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml new file mode 100644 index 0000000..0a83e12 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro-extension</artifactId> + <version>0.4.1</version> + </parent> + + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro-assembly</artifactId> + <version>0.4.1</version> + <packaging>pom</packaging> + <name>metron-parser-bro-assembly</name> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml new file mode 100644 index 0000000..d292a2d --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${project.basedir}/../metron-parser-bro/src/main/config</directory> + <outputDirectory>/config</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/../metron-parser-bro-bundle/target</directory> + <includes> + <include>metron-parser-bro-bundle-${project.version}.bundle</include> + </includes> + <outputDirectory>/lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml new file mode 100644 index 0000000..26d8b6e --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro-extension</artifactId> + <version>0.4.1</version> + </parent> + + <artifactId>metron-parser-bro-bundle</artifactId> + <version>0.4.1</version> + <name>metron-parser-bro-bundle</name> + <packaging>bundle</packaging> + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> + <source.skip>false</source.skip> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro</artifactId> + <version>0.4.1</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md new file mode 100644 index 0000000..2e8119a --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md @@ -0,0 +1,3 @@ +# bro Parser + +This is the bro parser http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml new file mode 100644 index 0000000..a22b52c --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml @@ -0,0 +1,83 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro-extension</artifactId> + <version>0.4.1</version> + </parent> + <artifactId>metron-parser-bro</artifactId> + <version>0.4.1</version> + <name>metron-parser-bro</name> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.atteo.classindex</groupId> + <artifactId>classindex</artifactId> + <version>${global_classindex_version}</version> + <scope>provided</scope> + </dependency> + <!-- testing --> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions-testing</artifactId> + <version>${project.parent.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/patterns</directory> + </resource> + <resource> + <directory>src/test/resources</directory> + </resource> + </resources> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template new file mode 100644 index 0000000..18c5d9b --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template @@ -0,0 +1,972 @@ +{ + "template": "bro_index*", + "mappings": { + "bro_doc": { + "_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "geo_location_point": { + "match": "enrichments:geo:*:location_point", + "match_mapping_type": "*", + "mapping": { + "type": "geo_point" + } + } + }, + { + "geo_country": { + "match": "enrichments:geo:*:country", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_city": { + "match": "enrichments:geo:*:city", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_location_id": { + "match": "enrichments:geo:*:locID", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_dma_code": { + "match": "enrichments:geo:*:dmaCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_postal_code": { + "match": "enrichments:geo:*:postalCode", + "match_mapping_type": "*", + "mapping": { + "type": "string", + "index": "not_analyzed" + } + } + }, + { + "geo_latitude": { + "match": "enrichments:geo:*:latitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "geo_longitude": { + "match": "enrichments:geo:*:longitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "timestamps": { + "match": "*:ts", + "match_mapping_type": "*", + "mapping": { + "type": "date", + "format": "epoch_millis" + } + } + }, + { + "threat_triage_score": { + "mapping": { + "type": "float" + }, + "match": "threat.triage.rules:*:score", + "match_mapping_type": "*" + } + }, + { + "threat_triage_reason": { + "mapping": { + "type": "string" + }, + "match": "threat.triage.rules:*:reason", + "match_mapping_type": "*" + } + }, + { + "threat_triage_name": { + "mapping": { + "type": "string" + }, + "match": "threat.triage.rules:*:name", + "match_mapping_type": "*" + } + } + ], + "properties": { + /* + * WARNING + * + * Because Metron inserts many distinct bro records into a single ElasticSearch index + * by default, it encounters field collisions due to field name reuse across bro logs. + * + * Be careful when modifying this file to not unintentionally affect other logs. + * For instance, the "version" field exists in the HTTP, SSL, and SSH logs. If you + * were to only consider the SSH log, you would set the type to integer, but because + * in the SSL and HTTP logs version is a string, we must set the type to string. + */ + /* + * Metron-specific fields + */ + "source:type": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Widely-used Bro fields (potentially renamed during Metron ingest) + */ + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "uid": { + "type": "string", + "index": "not_analyzed" + }, + "ip_src_addr": { + "type": "ip" + }, + "ip_src_port": { + "type": "integer" + }, + "ip_dst_addr": { + "type": "ip" + }, + "ip_dst_port": { + "type": "integer" + }, + /* + * HTTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info + * + * Notable Fields + * Field: password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: capture_password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: trans_depth + * Notes: Field exists in the HTTP and SMTP logs + * + * Field: user_agent + * Notes: Field exists in the HTTP and SMTP logs + * + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + * + * Field: host + * Notes: Field exists in the HTTP and Software logs + * + * Field: username + * Notes: Field exists in the HTTP and RADIUS logs + */ + "trans_depth": { + "type": "integer" + }, + "method": { + "type": "string", + "index": "not_analyzed" + }, + "host": { + "type": "string", + "index": "not_analyzed" + }, + "uri": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 8191 + }, + "referrer": { + "type": "string", + "index": "not_analyzed" + }, + "version": { + "type": "string", + "index": "not_analyzed" + }, + "user_agent": { + "type": "string" + }, + "request_body_len": { + "type": "long" + }, + "response_body_len": { + "type": "long" + }, + "status_code": { + "type": "integer" + }, + "status_msg": { + "type": "string", + "index": "not_analyzed" + }, + "username": { + "type": "string", + "index": "not_analyzed" + }, + "password": { + "type": "string", + "index": "not_analyzed" + }, + "capture_password": { + "type": "boolean" + }, + /* + * DNS log support + * https://www.bro.org/sphinx/scripts/base/protocols/dns/main.bro.html#type-DNS::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + * + * Field: trans_id + * Notes: Field exists in the DNS and DHCP logs + */ + "proto": { + "type": "string", + "index": "not_analyzed" + }, + "trans_id": { + "type": "long" + }, + "query": { + "type": "string", + "index": "not_analyzed" + }, + "qclass": { + "type": "integer" + }, + "qclass_name": { + "type": "string", + "index": "not_analyzed" + }, + "qtype": { + "type": "integer" + }, + "qtype_name": { + "type": "string", + "index": "not_analyzed" + }, + "rcode": { + "type": "integer" + }, + "rcode_name": { + "type": "string", + "index": "not_analyzed" + }, + "AA": { + "type": "boolean" + }, + "TC": { + "type": "boolean" + }, + "RD": { + "type": "boolean" + }, + "RA": { + "type": "boolean" + }, + "Z": { + "type": "integer" + }, + "answers": { + "type": "string" + }, + "rejected": { + "type": "boolean" + }, + /* + * Conn log support + * https://www.bro.org/sphinx/scripts/base/protocols/conn/main.bro.html#type-Conn::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + * + * Field: duration + * Notes: Field exists in the Conn and Files logs + * + * Field: local_orig + * Notes: Field exists in the Conn and Files logs + */ + "service": { + "type": "string", + "index": "not_analyzed" + }, + "duration": { + "type": "float" + }, + "orig_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "resp_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "conn_state": { + "type": "string", + "index": "not_analyzed" + }, + "local_orig": { + "type": "boolean" + }, + "local_resp": { + "type": "string", + "index": "not_analyzed" + }, + "missed_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "history": { + "type": "string", + "index": "not_analyzed" + }, + "orig_pkts": { + "type": "long", + "index": "not_analyzed" + }, + "orig_ip_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "resp_pkts": { + "type": "long", + "index": "not_analyzed" + }, + "resp_ip_bytes": { + "type": "long", + "index": "not_analyzed" + }, + "tunnel_parents": { + "type": "string", + "index": "not_analyzed" + }, + /* + * DPD log support + * https://www.bro.org/sphinx-git/scripts/base/frameworks/dpd/main.bro.html#type-DPD::Info + * + * Notable Fields + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + */ + "analyzer": { + "type": "string", + "index": "not_analyzed" + }, + "failure_reason": { + "type": "string", + "index": "not_analyzed" + }, + /* + * FTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/ftp/info.bro.html#type-FTP::Info + * + * Notable Fields + * Field: password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: capture_password + * Notes: Field exists in the HTTP and FTP logs + * + * Field: mime_type + * Notes: Field exists in the FTP and Files logs + * + * Field: fuid + * Notes: Field exists in the FTP and Notice logs + */ + "user": { + "type": "string", + "index": "not_analyzed" + }, + "command": { + "type": "string", + "index": "not_analyzed" + }, + "arg": { + "type": "string", + "analyzer": "simple" + }, + "mime_type": { + "type": "string", + "analyzer": "simple" + }, + "file_size": { + "type": "long" + }, + "reply_code": { + "type": "integer" + }, + "reply_msg": { + "type": "string", + "index": "not_analyzed" + }, + "data_channel:passive": { + "type": "boolean" + }, + "data_channel:orig_h": { + "type": "ip" + }, + "data_channel:resp_h": { + "type": "ip" + }, + "data_channel:resp_p": { + "type": "integer" + }, + "cwd": { + "type": "string", + "analyzer": "simple" + }, + "passive": { + "type": "boolean" + }, + "fuid": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Files log support + * https://www.bro.org/sphinx/scripts/base/frameworks/files/main.bro.html#type-Files::Info + * + * Notable Fields + * Field: tx_hosts + * Notes: Metron rewrites this to "ip_src_addr" + * + * Field: rx_hosts + * Notes: Metron rewrites this to "ip_dst_addr" + * + * Field: mime_type + * Notes: Field exists in the FTP and Files logs + */ + "conn_uids": { + "type": "string", + "analyzer": "simple" + }, + "source": { + "type": "string", + "index": "not_analyzed" + }, + "depth": { + "type": "integer" + }, + "analyzers": { + "type": "string", + "analyzer": "simple" + }, + "filename": { + "type": "string", + "index": "not_analyzed" + }, + "is_orig": { + "type": "boolean" + }, + "seen_bytes": { + "type": "long" + }, + "total_bytes": { + "type": "long" + }, + "missing_bytes": { + "type": "long" + }, + "overflow_bytes": { + "type": "long" + }, + "timedout": { + "type": "boolean" + }, + "parent_fuid": { + "type": "string", + "index": "not_analyzed" + }, + "md5": { + "type": "string", + "index": "not_analyzed" + }, + "sha1": { + "type": "string", + "index": "not_analyzed" + }, + "sha256": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Known::CertInfo log support + * https://www.bro.org/sphinx/scripts/policy/protocols/ssl/known-certs.bro.html#type-Known::CertsInfo + * + * Notable Fields + * Field: subject + * Notes: Field exists in the Known::CertInfo and SMTP logs + */ + "port_num": { + "type": "integer" + }, + "subject": { + "type": "string", + "analyzer": "simple" + }, + "issuer_subject": { + "type": "string", + "analyzer": "simple" + }, + "serial": { + "type": "string", + "index": "not_analyzed" + }, + /* + * SMTP log support + * https://www.bro.org/sphinx/scripts/base/protocols/smtp/main.bro.html#type-SMTP::Info + * + * Notable Fields + * Field: subject + * Notes: Field exists in the Known::CertInfo and SMTP logs + */ + "helo": { + "type": "string", + "analyzer": "simple" + }, + "mailfrom": { + "type": "string", + "analyzer": "simple" + }, + "rcptto": { + "type": "string", + "analyzer": "simple" + }, + "date": { + "type": "string", + "index": "not_analyzed" + }, + "from": { + "type": "string", + "analyzer": "simple" + }, + "to": { + "type": "string", + "analyzer": "simple" + }, + "reply_to": { + "type": "string", + "analyzer": "simple" + }, + "msg_id": { + "type": "string", + "index": "not_analyzed" + }, + "in_reply_to": { + "type": "string", + "index": "not_analyzed" + }, + "x_originating_ip": { + "type": "ip" + }, + "first_received": { + "type": "string", + "analyzer": "simple" + }, + "second_received": { + "type": "string", + "analyzer": "simple" + }, + "last_reply": { + "type": "string", + "analyzer": "simple" + }, + "path": { + "type": "string", + "index": "not_analyzed" + }, + "tls": { + "type": "boolean" + }, + "fuids": { + "type": "string", + "index": "not_analyzed" + }, + "is_webmail": { + "type": "boolean" + }, + /* + * SSL log support + * https://www.bro.org/sphinx/scripts/base/protocols/ssl/main.bro.html#type-SSL::Info + * + * Notable Fields + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + */ + "cipher": { + "type": "string", + "index": "not_analyzed" + }, + "curve": { + "type": "string", + "index": "not_analyzed" + }, + "server_name": { + "type": "string", + "index": "not_analyzed" + }, + "resumed": { + "type": "boolean" + }, + "last_alert": { + "type": "string", + "index": "not_analyzed" + }, + "next_protocol": { + "type": "string", + "index": "not_analyzed" + }, + "established": { + "type": "boolean" + }, + /* + * Weird log support + * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info + */ + "name": { + "type": "string", + "index": "not_analyzed" + }, + "addl": { + "type": "string", + "index": "not_analyzed" + }, + "notice": { + "type": "boolean" + }, + "peer": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Notice log support + * https://www.bro.org/sphinx/scripts/base/frameworks/notice/main.bro.html#type-Notice::Info + * + * Notable Fields + * Field: fuid + * Notes: Field exists in the FTP and Notice logs + * + * Field: proto + * Notes: Field exists in the DNS, Conn, DPD, and Notice logs + */ + "file_mime_type": { + "type": "string", + "index": "not_analyzed" + }, + "file_desc": { + "type": "string", + "index": "not_analyzed" + }, + "note": { + "type": "string", + "index": "not_analyzed" + }, + "msg": { + "type": "string", + "index": "not_analyzed" + }, + "sub": { + "type": "string", + "index": "not_analyzed" + }, + "src": { + "type": "ip" + }, + "dst": { + "type": "ip" + }, + "p": { + "type": "integer", + "index": "not_analyzed" + }, + "n": { + "type": "integer", + "index": "not_analyzed" + }, + "src_peer": { + "type": "ip" + }, + "peer_descr": { + "type": "string", + "index": "not_analyzed" + }, + "actions": { + "type": "string", + "index": "not_analyzed" + }, + "suppress_for": { + "type": "double", + "index": "not_analyzed" + }, + "dropped": { + "type": "boolean" + }, + /* + * DHCP log support + * https://www.bro.org/sphinx/scripts/base/protocols/dhcp/main.bro.html#type-DHCP::Info + * + * Notable Fields + * Field: trans_id + * Notes: Field exists in the DNS and DHCP logs + * + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "mac": { + "type": "string", + "index": "not_analyzed" + }, + "assigned_ip": { + "type": "ip" + }, + "lease_time": { + "type": "float", + "index": "not_analyzed" + }, + /* + * SSH log support + * https://www.bro.org/sphinx/scripts/base/protocols/ssh/main.bro.html#type-SSH::Info + * + * Notable Fields + * Field: version + * Notes: Field exists in the HTTP, SSL, and SSH logs + */ + "auth_success": { + "type": "boolean" + }, + "auth_attempts": { + "type": "integer", + "index": "not_analyzed" + }, + "direction": { + "type": "string", + "index": "not_analyzed" + }, + "client": { + "type": "string", + "index": "not_analyzed" + }, + "server": { + "type": "string", + "index": "not_analyzed" + }, + "cipher_alg": { + "type": "string", + "index": "not_analyzed" + }, + "mac_alg": { + "type": "string", + "index": "not_analyzed" + }, + "compression_alg": { + "type": "string", + "index": "not_analyzed" + }, + "kex_alg": { + "type": "string", + "index": "not_analyzed" + }, + "host_key_alg": { + "type": "string", + "index": "not_analyzed" + }, + "host_key": { + "type": "string", + "index": "not_analyzed" + }, + /* + * Software log support + * https://www.bro.org/sphinx/scripts/base/frameworks/software/main.bro.html#type-Software::Info + * + * Notable Fields + * Field: host + * Notes: Field exists in the HTTP and Software logs + */ + "host_p": { + "type": "integer", + "index": "not_analyzed" + }, + "software_type": { + "type": "string", + "index": "not_analyzed" + }, + "version:major": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor2": { + "type": "string", + "index": "not_analyzed" + }, + "version:minor3": { + "type": "string", + "index": "not_analyzed" + }, + "version:addl": { + "type": "string", + "index": "not_analyzed" + }, + "unparsed_version": { + "type": "string", + "analyzer": "simple" + }, + /* + * RADIUS log support + * https://www.bro.org/sphinx/scripts/base/protocols/radius/main.bro.html#type-RADIUS::Info + * + * Notable Fields + * Field: username + * Notes: Field exists in the HTTP and RADIUS logs + * + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "remote_ip": { + "type": "ip" + }, + "connect_info": { + "type": "string", + "index": "not_analyzed" + }, + "result": { + "type": "string", + "index": "not_analyzed" + }, + /* + * X509 log support + * https://www.bro.org/sphinx/scripts/base/files/x509/main.bro.html#type-X509::Info + * + * Notable Fields + * Field: id + * Notes: In other bro records, the id field is of type conn_id, so it is + * expanded before being logged into 4 fields, all of which are addressed + * under the "Widely-used Bro fields" section of this template. In X509 + * logs, however, id is a string to identify the certificate file id. + */ + "id": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:version": { + "type": "integer", + "index": "not_analyzed" + }, + "certificate:serial": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:subject": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:issuer": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:not_valid_before": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:not_valid_after": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_alg": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:sig_alg": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_type": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:key_length": { + "type": "integer", + "index": "not_analyzed" + }, + "certificate:exponent": { + "type": "string", + "index": "not_analyzed" + }, + "certificate:curve": { + "type": "string", + "index": "not_analyzed" + }, + "san:dns": { + "type": "string", + "index": "not_analyzed" + }, + "san:uri": { + "type": "string", + "index": "not_analyzed" + }, + "san:email": { + "type": "string", + "index": "not_analyzed" + }, + "san:ip": { + "type": "string", + "index": "not_analyzed" + }, + "basic_constraints:ca": { + "type": "boolean" + }, + "basic_constraints:path_len": { + "type": "integer", + "index": "not_analyzed" + }, + /* + * Known::DevicesInfo log support + * https://www.bro.org/sphinx/scripts/policy/misc/known-devices.bro.html#type-Known::DevicesInfo + * + * Notable Fields + * Field: mac + * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs + */ + "dhcp_host_name": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json new file mode 100644 index 0000000..824c812 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json @@ -0,0 +1,17 @@ +{ + "enrichment" : { + "fieldMap": { + "geo": ["ip_dst_addr", "ip_src_addr"], + "host": ["host"] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + }, + "fieldToTypeMap": { + "ip_src_addr" : ["malicious_ip"], + "ip_dst_addr" : ["malicious_ip"] + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json new file mode 100644 index 0000000..4ee131d --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json @@ -0,0 +1,18 @@ +{ + "hdfs" : { + "index": "bro", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "bro", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "bro", + "batchSize": 5, + "enabled" : true + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json new file mode 100644 index 0000000..7cbd0c1 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json @@ -0,0 +1,6 @@ +{ + "parserClassName": "org.apache.metron.parsers.bro.BasicBroParser", + "sensorTopic": "bro", + "parserConfig": { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java new file mode 100644 index 0000000..5264750 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.parsers.bro; + +import java.lang.invoke.MethodHandles; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.metron.common.Constants; +import org.apache.metron.parsers.BasicParser; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("serial") +public class BasicBroParser extends BasicParser { + + protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final ThreadLocal<NumberFormat> DECIMAL_FORMAT = new ThreadLocal<NumberFormat>() { + @Override + protected NumberFormat initialValue() { + return new DecimalFormat("0.0#####"); + } + }; + private JSONCleaner cleaner = new JSONCleaner(); + + @Override + public void configure(Map<String, Object> parserConfig) { + + } + + @Override + public void init() { + + } + + @Override + @SuppressWarnings("unchecked") + public List<JSONObject> parse(byte[] msg) { + + _LOG.trace("[Metron] Starting to parse incoming message"); + + String rawMessage = null; + List<JSONObject> messages = new ArrayList<>(); + try { + rawMessage = new String(msg, "UTF-8"); + _LOG.trace("[Metron] Received message: {}", rawMessage); + + JSONObject cleanedMessage = cleaner.clean(rawMessage); + _LOG.debug("[Metron] Cleaned message: {}", cleanedMessage); + + if (cleanedMessage == null || cleanedMessage.isEmpty()) { + throw new Exception("Unable to clean message: " + rawMessage); + } + + String key; + JSONObject payload; + if (cleanedMessage.containsKey("type")) { + key = cleanedMessage.get("type").toString(); + payload = cleanedMessage; + } else { + key = cleanedMessage.keySet().iterator().next().toString(); + + if (key == null) { + throw new Exception("Unable to retrieve key for message: " + + rawMessage); + } + + payload = (JSONObject) cleanedMessage.get(key); + } + + if (payload == null) { + throw new Exception("Unable to retrieve payload for message: " + + rawMessage); + } + + String originalString = key.toUpperCase() + " |"; + for (Object k : payload.keySet()) { + Object raw = payload.get(k); + String value = raw.toString(); + if (raw instanceof Double) { + value = DECIMAL_FORMAT.get().format(raw); + } + originalString += " " + k.toString() + ":" + value; + } + payload.put("original_string", originalString); + + replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[]{ "ts" }); + + long timestamp = 0L; + if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) { + try { + Double broTimestamp = ((Number) payload.get(Constants.Fields.TIMESTAMP.getName())).doubleValue(); + String broTimestampFormatted = DECIMAL_FORMAT.get().format(broTimestamp); + timestamp = convertToMillis(broTimestamp); + payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp); + payload.put("bro_timestamp", broTimestampFormatted); + _LOG.trace("[Metron] new bro record - timestamp : {}", payload.get(Constants.Fields.TIMESTAMP.getName())); + } catch (NumberFormatException nfe) { + _LOG.error("[Metron] timestamp is invalid: {}", payload.get("timestamp")); + payload.put(Constants.Fields.TIMESTAMP.getName(), 0); + } + } + + boolean ipSrcReplaced = replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{"source_ip", "id.orig_h"}); + if (!ipSrcReplaced) { + replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{ "tx_hosts" }); + } + + boolean ipDstReplaced = replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[]{"dest_ip", "id.resp_h"}); + if (!ipDstReplaced) { + replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[]{ "rx_hosts" }); + } + + replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[]{"source_port", "id.orig_p"}); + replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[]{"dest_port", "id.resp_p"}); + + payload.put(Constants.Fields.PROTOCOL.getName(), key); + _LOG.debug("[Metron] Returning parsed message: {}", payload); + messages.add(payload); + return messages; + + } catch (Exception e) { + String message = "Unable to parse Message: " + rawMessage; + _LOG.error(message, e); + throw new IllegalStateException(message, e); + } + + } + + private Long convertToMillis(Double timestampSeconds) { + return ((Double) (timestampSeconds * 1000)).longValue(); + } + + private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) { + for (String fromKey : fromKeys) { + if (payload.containsKey(fromKey)) { + Object value = payload.remove(fromKey); + payload.put(toKey, value); + _LOG.trace("[Metron] Added {} to {}", toKey, payload); + return true; + } + } + return false; + } + + private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) { + for (String fromKey : fromKeys) { + if (payload.containsKey(fromKey)) { + JSONArray value = (JSONArray) payload.remove(fromKey); + if (value != null && !value.isEmpty()) { + payload.put(toKey, value.get(0)); + _LOG.trace("[Metron] Added {} to {}", toKey, payload); + return true; + } + } + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java new file mode 100644 index 0000000..b3647d5 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.bro; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class JSONCleaner implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 1L; + + + /** + * @param jsonString + * @return + * @throws ParseException + * Takes a json String as input and modifies the keys to remove any characters other than . _ a-z A-Z or 0-9 + */ + @SuppressWarnings({"unchecked","rawtypes"}) + public JSONObject clean(String jsonString) throws ParseException + { + JSONParser parser = new JSONParser(); + + + Map json = (Map) parser.parse(jsonString); + JSONObject output = new JSONObject(); + Iterator iter = json.entrySet().iterator(); + + while(iter.hasNext()){ + Map.Entry entry = (Map.Entry)iter.next(); + + String key = ((String)entry.getKey()).replaceAll("[^\\._a-zA-Z0-9]+",""); + output.put(key, entry.getValue()); + } + + return output; + } + + + @SuppressWarnings({ "unchecked", "rawtypes", "unused" }) + public static void main(String args[]) + { + String jsonText = "{\"first_1\": 123, \"second\": [4, 5, 6], \"third\": 789}"; + JSONCleaner cleaner = new JSONCleaner(); + try { + //cleaner.clean(jsonText); + Map obj=new HashMap(); + obj.put("name","foo"); + obj.put("num", 100); + obj.put("balance", 1000.21); + obj.put("is_vip", true); + obj.put("nickname",null); + Map obj1 = new HashMap(); + obj1.put("sourcefile", obj); + + JSONObject json = new JSONObject(obj1); + System.out.println(json); + + + + System.out.print(jsonText); + } catch (Exception e) { + e.printStackTrace(); + } + } + +}
