This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 04593a6 METRON-1911 Create Docker based test environment for Bro
Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21
04593a6 is described below
commit 04593a6a2d6ee4b3b1d43aab70ad4a406be240fc
Author: ottobackwards <ottobackwa...@gmail.com>
AuthorDate: Wed Feb 6 09:25:51 2019 -0500
METRON-1911 Create Docker based test environment for Bro Kafka Plugin
(ottobackwards) closes apache/metron-bro-plugin-kafka#21
---
.gitignore | 12 +
NOTICE | 4 +-
docker/README.md | 291 +++++++++++++++++++++
.../containers/bro-localbuild-container/Dockerfile | 49 ++++
docker/data/.gitignore | 4 +
docker/finish_end_to_end.sh | 35 +++
docker/in_docker_scripts/build_bro_plugin.sh | 43 +++
docker/in_docker_scripts/configure_bro_plugin.sh | 41 +++
docker/in_docker_scripts/process_data_file.sh | 78 ++++++
docker/in_docker_scripts/wait-for-it.sh | 162 ++++++++++++
docker/in_docker_scripts/wait_for_kafka.sh | 35 +++
docker/in_docker_scripts/wait_for_zk.sh | 35 +++
docker/remove_timeout_message.sh | 51 ++++
docker/run_end_to_end.sh | 202 ++++++++++++++
docker/scripts/build_container.sh | 111 ++++++++
docker/scripts/cleanup_docker.sh | 91 +++++++
docker/scripts/create_docker_network.sh | 79 ++++++
docker/scripts/destroy_docker_network.sh | 76 ++++++
docker/scripts/docker_execute_build_bro_plugin.sh | 84 ++++++
.../scripts/docker_execute_configure_bro_plugin.sh | 84 ++++++
docker/scripts/docker_execute_process_data_file.sh | 113 ++++++++
docker/scripts/docker_execute_shell.sh | 79 ++++++
docker/scripts/docker_run_bro_container.sh | 168 ++++++++++++
docker/scripts/docker_run_consume_bro_kafka.sh | 86 ++++++
.../docker_run_create_bro_topic_in_kafka.sh | 74 ++++++
docker/scripts/docker_run_get_offset_bro_kafka.sh | 75 ++++++
docker/scripts/docker_run_kafka_container.sh | 82 ++++++
docker/scripts/docker_run_wait_for_kafka.sh | 82 ++++++
docker/scripts/docker_run_wait_for_zookeeper.sh | 82 ++++++
docker/scripts/docker_run_zookeeper_container.sh | 82 ++++++
docker/scripts/download_sample_pcaps.sh | 112 ++++++++
docker/scripts/print_results.sh | 91 +++++++
docker/scripts/split_kakfa_output_by_log.sh | 106 ++++++++
docker/scripts/stop_container.sh | 85 ++++++
docker/test_output/.gitignore | 4 +
35 files changed, 2887 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 28a8358..735a17e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,9 @@
+#ide stuff
+.idea
+*.iml
+*.iws
+.DS_Store
+
.state
build
@@ -29,3 +35,9 @@ build
*.exe
*.out
*.app
+
+# Log files
+*.log
+# pcap files
+*.pcap
+*.pcapng
diff --git a/NOTICE b/NOTICE
index 9505518..7f966de 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,7 @@
Apache Metron
- Copyright 2015-2016 The Apache Software Foundation
+ Copyright 2015-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+
+ This product bundles wait-for-it.sh, which is available under a "MIT
Software License" license. For details, see
https://github.com/vishnubob/wait-for-it
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000..8e4d3fa
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,291 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+## Docker support for testing metron-bro-plugin-kafka
+
+These scripts and containers provide support for building and testing Bro and
the metron-bro-plugin-kafka using a number of Docker containers.
+The use of these scripts and containers allow an easier, automated workflow
for testing new features, fixes, or regressions than before.
+One of the goals is for this to be extensible, such that new scripts can be
introduced and run as well. This will allow, for example, one or more
+testing scripts to be added to a pull request, and subsequently to a test
suite.
+
+
+#### Directories
+
+```bash
+├── containers
+│ └── bro-localbuild-container
+├── data
+├── in_docker_scripts
+├── scripts
+└── test_output
+```
+- `containers`: The parent of all of the containers that this project defines.
We use several containers, not all of them ours.
+ - `bro-localbuild-container`: The docker container directory for our bro
container, used for building bro, the librdkafka, and our plugin, as well as
running bro.
+- `data`: The default path for pcap data to be used in tests.
+- `in_docker_scripts`: This directory is mapped to the bro docker container as
/root/built_in_scripts. These represent the library of scripts we provide to
be run in the docker container.
+- `scripts`: These are the scripts that are run on the host for creating the
docker bits, running containers, running or executing commands against
containers ( such as executing one of the built_in_scripts ), and cleaning up
resources.
+- `test_output`: Directory where the bro logs and kafka logs per test/pcap are
stored.
+
+
+#### Scripts that execute _in_ the docker container
+
+```bash
+├── build_bro_plugin.sh
+├── configure_bro_plugin.sh
+├── process_data_file.sh
+├── wait-for-it.sh
+├── wait_for_kafka.sh
+└── wait_for_zk.sh
+```
+
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `configure_bro_plugin.sh`: Configures the plugin for the kafka container,
and routes all traffic types.
+- `process_data_file.sh`: Runs `bro -r` on the passed file
+- `wait-for-it.sh`: Waits for a port to be open, so we know something is
available.
+- `wait_for_kafka.sh`: Waits for the kafka to be available.
+- `wait_for_zk.sh`: Waits for zookeeper to be available.
+
+
+#### Scripts executed on the host to setup and interact with the docker
containers
+
+```bash
+├── build_container.sh
+├── cleanup_docker.sh
+├── create_docker_network.sh
+├── destroy_docker_network.sh
+├── docker_execute_build_bro_plugin.sh
+├── docker_execute_configure_bro_plugin.sh
+├── docker_execute_process_data_file.sh
+├── docker_execute_shell.sh
+├── docker_run_bro_container.sh
+├── docker_run_consume_bro_kafka.sh
+├── docker_run_create_bro_topic_in_kafka.sh
+├── docker_run_get_offset_bro_topic_in_kafka.sh
+├── docker_run_kafka_container.sh
+├── docker_run_wait_for_kafka.sh
+├── docker_run_wait_for_zookeeper.sh
+├── docker_run_zookeeper_container.sh
+├── download_sample_pcaps.sh
+├── print_results.sh
+├── split_kakfa_output_by_log.sh
+└── stop_container.sh
+```
+
+- `build_container.sh`: Runs docker build in the passed directory, and names
the results
+ ###### Parameters
+ ```bash
+ --container-directory [REQUIRED] The directory with the Dockerfile
+ --container-name [REQUIRED] The name to give the Docker
container
+ ```
+- `cleanup_containers.sh`: Stops the containers and destroys the network
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `create_docker_network.sh`: Creates the Docker network that the containers
will use
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `destroy_docker_network.sh`: Destroys a Docker network by calling `docker
network rm`
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `docker_execute_build_bro_plugin.sh`: Executes `build_bro_plugin.sh` in the
bro container
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ ```
+- `docker_execute_configure_bro_plugin.sh`: Executes `configure_bro_plugin.sh`
in the bro container
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ ```
+- `docker_execute_process_data_dir.sh`: Executes `process_data_dir.sh` in the
bro container
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ ```
+- `docker_execute_shell.sh`: `docker execute -i -t bash` to get a shell in a
given container
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ ```
+- `docker_run_bro_container.sh`: Runs the bro docker container in the
background
+ ###### Parameters
+ ```bash
+ --container-name [OPTIONAL] The Docker container name.
Default: bro
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ --scripts-path [OPTIONAL] The path with the scripts you may
run in the container. These are your scripts, not the built in scripts
+ --data-path [OPTIONAL] The name of the directory to map
to /root/data in the container
+ --docker-parameter [OPTIONAL, MULTIPLE] Each parameter with
this name will be passed to docker run
+ ```
+
+ > NOTE about `--scripts-path`
+ > The scripts path provided with be mapped into the bro container at
`/root/scripts`. This allows you to _inject_ your own scripts (not managed as
part of this source project) into the container.
+ > You can then execute these scripts or use them together as part of testing
etc. by creating `docker execute` scripts like those here.
+ > The goal is to allow an individual to use and maintain their own library
of scripts to use instead of, or in concert with the scripts maintained by this
project.
+
+- `docker_run_consume_bro_kafka.sh`: Runs an instance of the kafka container,
with the console consumer `kafka-console-consumer.sh --topic bro --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ --offset [OPTIONAL] The kafka offset. Default: -1
+ ```
+- `docker_run_get_offset_bro_kafka.sh`: Runs an instance of the kafka
container and gets the current offset for the bro topic
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ --offset [OPTIONAL] The kafka offset. Default: -1
+ ```
+- `docker_run_create_bro_topic_in_kafka.sh`: Runs an instance of the kafka
container, creating the `bro` topic
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `docker_run_kafka_container.sh`: Runs the main instance of the kafka
container in the background
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `docker_run_wait_for_kafka.sh`: Runs the `wait_for_kafka.sh` in a base
centos container
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `docker_run_wait_for_zookeeper.sh`: Runs the `wait_for_zk.sh` in a base
centos container
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `docker_run_zookeeper_container.sh`: Runs the zookeeper container in the
background
+ ###### Parameters
+ ```bash
+ --network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ ```
+- `download_sample_pcaps.sh`: Downloads the sample pcaps to a specified
directory. If they exist, it is a no-op
+
+ > The sample pcaps are:
+ > - https://www.bro.org/static/traces/exercise-traffic.pcap
+ > -
http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap
+ > - https://www.bro.org/static/traces/ssh.pcap
+ > -
https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true
+ > -
https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true
+ > - https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc
+
+ ###### Parameters
+ ```bash
+ --data-path [REQUIRED] The pcap data path
+ ```
+- `print_results.sh` : Prints the `results.csv` for all the pcaps processed in
the given directory to console
+ ###### Parameters
+ ```bash
+ --test-directory [REQUIRED] The directory for the tests
+ ```
+- `split_kafka_output_by_log.sh` : For a pcap result directory, will create a
LOG.kafka.log for each LOG.log's entry in the kafka-output.log
+ ###### Parameters
+ ```bash
+ --log-directory [REQUIRED] The directory with the logs
+ ```
+- `stop_container.sh`: Stops and removes a Docker container with a given name
+ ###### Parameters
+ ```bash
+ --container-name [REQUIRED] The Docker container name
+ ```
+
+#### The example end to end test script
+
+`run_end_to_end.sh` is provided as an example of a testing script. Specific
or extended scripts can be created similar to this script to use the containers.
+This script does the following:
+
+1. Creates the Docker network
+2. Runs the zookeeper container
+3. Waits for zookeeper to be available
+4. Runs the kafka container
+5. Waits for kafka to be available
+6. Creates the bro topic
+7. Downloads sample PCAP data
+8. Runs the bro container in the background
+
+> Note that all parameters passed to this script are passed to the
`docker_run_bro_container.sh` script
+
+9. Builds the bro plugin
+10. Configures the bro plugin
+11. Runs bro against all the pcap data, one at a time
+12. Executes a kafka client to read the data from bro for each pcap file
+13. Stores the output kafka messages and the bro logs into the test_output
directory
+
+```bash
+>tree Tue_Jan__8_21_54_10_EST_2019
+Tue_Jan__8_21_54_10_EST_2019
+├── exercise-traffic_pcap
+│ ├── capture_loss.log
+│ ├── conn.log
+│ ├── dhcp.log
+│ ├── dns.log
+│ ├── files.log
+│ ├── http.log
+│ ├── kafka-output.log
+│ ├── known_certs.log
+│ ├── known_devices.log
+│ ├── loaded_scripts.log
+│ ├── notice.log
+│ ├── packet_filter.log
+│ ├── reporter.log
+│ ├── smtp.log
+│ ├── software.log
+│ ├── ssl.log
+│ ├── stats.log
+│ ├── weird.log
+│ └── x509.log
+├── ftp_pcap
+│ ├── capture_loss.log
+│ ├── conn.log
+│ ├── files.log
+│ ├── ftp.log
+│ ├── kafka-output.log
+│ ├── loaded_scripts.log
+│ ├── packet_filter.log
+│ ├── reporter.log
+│ ├── software.log
+│ └── stats.log
+```
+
+14. Creates a results.csv for each pcap that has the line counts of the kafka
and the bro output for each log
+15. Prints all the results.csv to the screen
+
+As we can see, the output is a folder named for the test run time, with a sub
folder per pcap, containing all the bro logs and the kafka_output.log.
+
+
+At this point the containers are up and running in the background.
+
+Other scripts may then be used to do your testing, for example running:
+```bash
+./scripts/docker_execute_shell.sh
+```
+
+> NOTE: If the scripts are run repeatedly, and there is no change in bro or
the librdkafka, the line `./run_end_to_end.sh` can be replaced by
`./run_end_to_end.sh --skip-docker-build`, which uses the `--skip-docker-build`
flag to not rebuild the bro container, saving the time of rebuilding bro and
librdkafka.
+
+> NOTE: After you are done, you must call the `finish_end_to_end.sh` script to
cleanup.
+
+
+##### `run_end_to_end.sh`
+###### Parameters
+```bash
+--skip-docker-build [OPTIONAL] Skip build of bro docker machine.
+--data-path [OPTIONAL] The pcap data path. Default: ./data
+```
diff --git a/docker/containers/bro-localbuild-container/Dockerfile
b/docker/containers/bro-localbuild-container/Dockerfile
new file mode 100644
index 0000000..f7ad66f
--- /dev/null
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM centos:7
+WORKDIR /root
+
+# install development tools
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel
openssl-devel python-devel swig zlib-devel perl cyrus-sasl cyrus-sasl-devel
cyrus-sasl-gssapi git jq
+
+# install bro
+RUN curl -L https://www.bro.org/downloads/bro-2.5.5.tar.gz | tar xvz
+WORKDIR bro-2.5.5/
+RUN ./configure
+RUN make
+RUN make install
+ENV PATH="${PATH}:/usr/local/bro/bin"
+
+# install pip
+RUN yum -y update
+RUN yum -y install epel-release
+RUN yum -y install python-pip
+RUN pip install --upgrade pip
+
+# install bro-pkg
+RUN pip install bro-pkg
+ENV PATH="${PATH}:/usr/bin"
+RUN bro-pkg autoconfig
+
+# install librdkafka
+RUN curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz |
tar xvz
+WORKDIR librdkafka-0.11.5/
+RUN ./configure --enable-sasl
+RUN make
+RUN make install
+WORKDIR /root
diff --git a/docker/data/.gitignore b/docker/data/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/docker/data/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
diff --git a/docker/finish_end_to_end.sh b/docker/finish_end_to_end.sh
new file mode 100755
index 0000000..1c43d79
--- /dev/null
+++ b/docker/finish_end_to_end.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This script should be run _after_ run_end_to_end.sh when you are finished
with your testing and the containers.
+# Do not run this if you plan on running docker_execute_shell.sh for example
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+# Stop all the containers and remote the networks
+bash "${ROOT_DIR}"/scripts/cleanup_docker.sh
+
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh
b/docker/in_docker_scripts/build_bro_plugin.sh
new file mode 100755
index 0000000..46277bd
--- /dev/null
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Runs bro-pkg to build and install the plugin
+#
+
+cd /root || exit 1
+
+echo "================================"
+
+bro-pkg install code --force
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR running bro-pkg install ${rc}"
+ exit ${rc}
+fi
+
+echo "================================"
+echo ""
+
+bro -N Apache::Kafka
+
+echo "================================"
+echo ""
+
diff --git a/docker/in_docker_scripts/configure_bro_plugin.sh
b/docker/in_docker_scripts/configure_bro_plugin.sh
new file mode 100755
index 0000000..74551c2
--- /dev/null
+++ b/docker/in_docker_scripts/configure_bro_plugin.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Configures the bro kafka plugin
+# Configures the kafka broker
+# Configures the plugin for all the traffic types
+#
+
+echo "Configuring kafka plugin"
+{
+ echo "@load packages"
+ echo "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG,
DPD::LOG, FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG,
Weird::LOG, Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG,
X509::LOG, Known::DEVICES_LOG, RFB::LOG, Stats::LOG, CaptureLoss::LOG,
SIP::LOG);"
+ echo "redef Kafka::topic_name = \"bro\";"
+ echo "redef Kafka::tag_json = T;"
+ echo "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] =
\"kafka:9092\");"
+ echo "redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);"
+ echo "redef Known::cert_tracking = ALL_HOSTS;"
+ echo "redef Software::asset_tracking = ALL_HOSTS;"
+} >> /usr/local/bro/share/bro/site/local.bro
+
+sed -i '86 a @load policy/protocols/dhcp/known-devices-and-hostnames.bro'
/usr/local/bro/share/bro/site/local.bro
+
diff --git a/docker/in_docker_scripts/process_data_file.sh
b/docker/in_docker_scripts/process_data_file.sh
new file mode 100755
index 0000000..021c5fd
--- /dev/null
+++ b/docker/in_docker_scripts/process_data_file.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2010
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+PCAP_FILE_NAME=
+OUTPUT_DIRECTORY_NAME=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # PCAP_FILE_NAME
+ #
+ # --pcap-file-name
+ #
+ --pcap-file-name=*)
+ PCAP_FILE_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # OUTPUT_DIRECTORY_NAME
+ #
+ # --output-directory-name
+ #
+ --output-directory-name=*)
+ OUTPUT_DIRECTORY_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "PCAP_FILE_NAME = ${PCAP_FILE_NAME}"
+echo "OUTPUT_DIRECTORY_NAME = ${OUTPUT_DIRECTORY_NAME}"
+
+cd /root || exit 1
+echo "================================"
+if [ ! -d /root/data ]; then
+ echo "DATA_PATH has not been set and mapped"
+ exit 1
+fi
+cd /root/test_output/"${OUTPUT_DIRECTORY_NAME}" || exit 1
+find /root/data -type f -name "${PCAP_FILE_NAME}" -exec echo "processing" '{}'
\; -exec bro -r '{}' /usr/local/bro/share/bro/site/local.bro -C \;
+echo "done with ${PCAP_FILE_NAME}"
diff --git a/docker/in_docker_scripts/wait-for-it.sh
b/docker/in_docker_scripts/wait-for-it.sh
new file mode 100755
index 0000000..73abf05
--- /dev/null
+++ b/docker/in_docker_scripts/wait-for-it.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2064,SC2124,SC2206,SC2086
+# Use this script to test if a given TCP host/port are available
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+ cat << USAGE >&2
+Usage:
+ $cmdname host:port [-s] [-t timeout] [-- command args]
+ -h HOST | --host=HOST Host or IP under test
+ -p PORT | --port=PORT TCP port under test
+ Alternatively, you specify the host and port
as host:port
+ -s | --strict Only execute subcommand if the test succeeds
+ -q | --quiet Don't output any status messages
+ -t TIMEOUT | --timeout=TIMEOUT
+ Timeout in seconds, zero for no timeout
+ -- COMMAND ARGS Execute command with args after the test
finishes
+USAGE
+ exit 1
+}
+
+wait_for()
+{
+ if [[ $TIMEOUT -gt 0 ]]; then
+ echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+ else
+ echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+ fi
+ start_ts=$(date +%s)
+ while :
+ do
+ (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+ result=$?
+ if [[ $result -eq 0 ]]; then
+ end_ts=$(date +%s)
+ echoerr "$cmdname: $HOST:$PORT is available after $((end_ts -
start_ts)) seconds"
+ break
+ fi
+ sleep 1
+ done
+ return $result
+}
+
+wait_for_wrapper()
+{
+ # In order to support SIGINT during timeout:
http://unix.stackexchange.com/a/57692
+ if [[ $QUIET -eq 1 ]]; then
+ timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT
--timeout=$TIMEOUT &
+ else
+ timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT
--timeout=$TIMEOUT &
+ fi
+ PID=$!
+ trap "kill -INT -$PID" INT
+ wait $PID
+ RESULT=$?
+ if [[ $RESULT -ne 0 ]]; then
+ echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for
$HOST:$PORT"
+ fi
+ return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+ case "$1" in
+ *:* )
+ hostport=(${1//:/ })
+ HOST=${hostport[0]}
+ PORT=${hostport[1]}
+ shift 1
+ ;;
+ --child)
+ CHILD=1
+ shift 1
+ ;;
+ -q | --quiet)
+ QUIET=1
+ shift 1
+ ;;
+ -s | --strict)
+ STRICT=1
+ shift 1
+ ;;
+ -h)
+ HOST="$2"
+ if [[ $HOST == "" ]]; then break; fi
+ shift 2
+ ;;
+ --host=*)
+ HOST="${1#*=}"
+ shift 1
+ ;;
+ -p)
+ PORT="$2"
+ if [[ $PORT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --port=*)
+ PORT="${1#*=}"
+ shift 1
+ ;;
+ -t)
+ TIMEOUT="$2"
+ if [[ $TIMEOUT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --timeout=*)
+ TIMEOUT="${1#*=}"
+ shift 1
+ ;;
+ --)
+ shift
+ CLI="$@"
+ break
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ echoerr "Unknown argument: $1"
+ usage
+ ;;
+ esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+ echoerr "Error: you need to provide a host and port to test."
+ usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+if [[ $CHILD -gt 0 ]]; then
+ wait_for
+ RESULT=$?
+ exit $RESULT
+else
+ if [[ $TIMEOUT -gt 0 ]]; then
+ wait_for_wrapper
+ RESULT=$?
+ else
+ wait_for
+ RESULT=$?
+ fi
+fi
+
+if [[ $CLI != "" ]]; then
+ if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+ echoerr "$cmdname: strict mode, refusing to execute subprocess"
+ exit $RESULT
+ fi
+ exec $CLI
+else
+ exit $RESULT
+fi
diff --git a/docker/in_docker_scripts/wait_for_kafka.sh
b/docker/in_docker_scripts/wait_for_kafka.sh
new file mode 100755
index 0000000..56045f4
--- /dev/null
+++ b/docker/in_docker_scripts/wait_for_kafka.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Waits until kafka is available on the default port
+#
+
+cd /root || exit 1
+
+if [ ! -d /root/scripts ]; then
+ echo "DOCKER_SCRIPTS_PATH has not been set and mapped"
+ exit 1
+fi
+
+cd /root/scripts || exit 1
+bash ./wait-for-it.sh kafka:9092
+
diff --git a/docker/in_docker_scripts/wait_for_zk.sh
b/docker/in_docker_scripts/wait_for_zk.sh
new file mode 100755
index 0000000..db8c1ff
--- /dev/null
+++ b/docker/in_docker_scripts/wait_for_zk.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Waits until zookeeper is available on the default port
+#
+
+cd /root || exit 1
+
+if [ ! -d /root/scripts ]; then
+ echo "DOCKER_SCRIPTS_PATH has not been set and mapped"
+ exit 1
+fi
+
+cd /root/scripts || exit 1
+bash ./wait-for-it.sh zookeeper:2181
+
diff --git a/docker/remove_timeout_message.sh b/docker/remove_timeout_message.sh
new file mode 100755
index 0000000..dc34f2c
--- /dev/null
+++ b/docker/remove_timeout_message.sh
@@ -0,0 +1,51 @@
+#! /usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Remove the exception text from piped input when we have purposefully timed
+# out reading kafka
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+LAST_CMD=
+SKIP_EXCEPTION_TEXT=false
+
+while read -r CMD; do
+ if [[ ${CMD} =~ ('ERROR Error processing message') ]]; then
+ LAST_CMD=${CMD}
+ elif [[ ${CMD} =~ ('kafka.consumer.ConsumerTimeoutException') ]]; then
+ SKIP_EXCEPTION_TEXT=true
+ elif [[ "$SKIP_EXCEPTION_TEXT" = true ]]; then
+ if [[ ! ${CMD} =~ (^at) ]]; then
+ echo "${CMD}"
+ fi
+ else
+ if [[ -n "$LAST_CMD" ]]; then
+ LAST_CMD=
+ fi
+ if [[ ! ${CMD} =~ (^--) ]]; then
+ echo "${CMD}"
+ fi
+ fi
+done
+
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
new file mode 100755
index 0000000..950dc60
--- /dev/null
+++ b/docker/run_end_to_end.sh
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --skip-docker-build [OPTIONAL] Skip build of bro
docker machine."
+ echo " --data-path [OPTIONAL] The pcap data path.
Default: ./data"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+SKIP_REBUILD_BRO=false
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+SCRIPT_DIR="${ROOT_DIR}"/scripts
+CONTAINER_DIR="${ROOT_DIR}"/containers/bro-localbuild-container
+DATA_PATH="${ROOT_DIR}"/data
+DATE=$(date)
+LOG_DATE=${DATE// /_}
+TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
+# Handle command line options
+for i in "$@"; do
+ case $i in
+
+ #
+ # SKIP_REBUILD_BRO
+ #
+ # --skip-docker-build
+ #
+ --skip-docker-build)
+ SKIP_REBUILD_BRO=true
+ shift # past argument
+ ;;
+
+ #
+ # DATA_PATH
+ #
+ --data-path=*)
+ DATA_PATH="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+ esac
+done
+
+EXTRA_ARGS="$*"
+
+echo "Running build_container with "
+echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO"
+echo "==================================================="
+
+# Create the network
+bash "${SCRIPT_DIR}"/create_docker_network.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Run the zookeeper container
+bash "${SCRIPT_DIR}"/docker_run_zookeeper_container.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Wait for zookeeper to be up
+bash "${SCRIPT_DIR}"/docker_run_wait_for_zookeeper.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Run the kafka container
+bash "${SCRIPT_DIR}"/docker_run_kafka_container.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Wait for kafka to be up
+bash "${SCRIPT_DIR}"/docker_run_wait_for_kafka.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Create the bro topic
+bash "${SCRIPT_DIR}"/docker_run_create_bro_topic_in_kafka.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Build the bro container
+if [[ "$SKIP_REBUILD_BRO" = false ]]; then
+ bash "${SCRIPT_DIR}"/build_container.sh \
+ --container-directory="${CONTAINER_DIR}" \
+ --container-name=metron-bro-docker-container:latest
+
+ rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+ fi
+fi
+
+# Download the pcaps
+bash "${SCRIPT_DIR}"/download_sample_pcaps.sh --data-path="${DATA_PATH}"
+
+mkdir "${TEST_OUTPUT_PATH}" || exit 1
+
+# Run the bro container and optionally the passed script _IN_ the container
+bash "${SCRIPT_DIR}"/docker_run_bro_container.sh \
+ --data-path="${DATA_PATH}" \
+ --test-output-path="${TEST_OUTPUT_PATH}" \
+ "$EXTRA_ARGS"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+# Build the bro plugin
+bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO BUILD PLUGIN. CHECK LOGS ${rc}"
+ exit ${rc}
+fi
+
+# Configure it the bro plugin
+bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO CONFIGURE PLUGIN. CHECK LOGS ${rc}"
+ exit ${rc}
+fi
+
+
+# for each pcap in the data directory, we want to
+# run bro then read the output from kafka
+# and output both of them to the same directory named
+# for the date/pcap
+
+
+for file in "${DATA_PATH}"/**/*.pcap*
+do
+ # get the file name
+ BASE_FILE_NAME=$(basename "${file}")
+ DOCKER_DIRECTORY_NAME=${BASE_FILE_NAME//\./_}
+
+ mkdir "${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" || exit 1
+ echo "MADE ${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+
+ # get the current offset in kafka
+ # this is where we are going to _start_
+ OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_bro_kafka.sh | sed
's/^bro:0:\(.*\)$/\1/')
+ echo "OFFSET------------------> ${OFFSET}"
+
+ bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh
--pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
+
+ rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO PROCESS ${file} DATA. CHECK LOGS, please run the
finish_end_to_end.sh when you are done."
+ exit ${rc}
+ fi
+
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
+ bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET |
"${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
+
+ rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
+ fi
+
+ "${SCRIPT_DIR}"/split_kakfa_output_by_log.sh
--log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+done
+
+"${SCRIPT_DIR}"/print_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+
+echo ""
+echo "Run complete"
+echo "The kafka and bro output can be found at ${TEST_OUTPUT_PATH}"
+echo "You may now work with the containers if you will. You need to call
finish_end_to_end.sh when you are done"
diff --git a/docker/scripts/build_container.sh
b/docker/scripts/build_container.sh
new file mode 100755
index 0000000..d4e5dca
--- /dev/null
+++ b/docker/scripts/build_container.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+#
+# Runs docker build in a provided directory, with a provided name
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-directory [REQUIRED] The directory with the
Dockerfile"
+ echo " --container-name [REQUIRED] The name to give the
Docker container"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+CONTAINER_DIRECTORY=
+CONTAINER_NAME=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_DIRECTORY
+ #
+ # --container-directory
+ #
+ --container-directory=*)
+ CONTAINER_DIRECTORY="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "$CONTAINER_DIRECTORY" ]]; then
+ echo "CONTAINER_DIRECTORY must be passed"
+ exit 1
+fi
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+ echo "CONTAINER_NAME must be passed"
+ exit 1
+fi
+
+echo "Running with "
+echo "CONTAINER_DIRECTORY = $CONTAINER_DIRECTORY"
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${CONTAINER_DIRECTORY}" || exit 1
+echo "==================================================="
+echo "docker build of ${CONTAINER_NAME}"
+echo "==================================================="
+
+docker build . --no-cache --tag="${CONTAINER_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
diff --git a/docker/scripts/cleanup_docker.sh b/docker/scripts/cleanup_docker.sh
new file mode 100755
index 0000000..5a95e3a
--- /dev/null
+++ b/docker/scripts/cleanup_docker.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+#
+# Stops the containers, and shuts down the NETWORK_NAME
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+CONTAINER_NAME=bro
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument
+ ;;
+
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+ esac
+done
+
+echo "Running cleanup_containers with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name="${CONTAINER_NAME}"
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name=kafka
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name=zookeeper
+
+"${SCRIPT_DIR}"/destroy_docker_network.sh --network-name="${NETWORK_NAME}"
+
diff --git a/docker/scripts/create_docker_network.sh
b/docker/scripts/create_docker_network.sh
new file mode 100755
index 0000000..efa1684
--- /dev/null
+++ b/docker/scripts/create_docker_network.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running create_docker_network with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker network create "${NETWORK_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+echo "Created network ${NETWORK_NAME}"
+
diff --git a/docker/scripts/destroy_docker_network.sh
b/docker/scripts/destroy_docker_network.sh
new file mode 100755
index 0000000..516c33c
--- /dev/null
+++ b/docker/scripts/destroy_docker_network.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information"
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running destroy_docker_network with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker network rm "${NETWORK_NAME}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_execute_build_bro_plugin.sh
b/docker/scripts/docker_execute_build_bro_plugin.sh
new file mode 100755
index 0000000..649fd4e
--- /dev/null
+++ b/docker/scripts/docker_execute_build_bro_plugin.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the build_bro_plugin.sh script in the container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+CONTAINER_NAME=bro
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running build_bro_plugin_docker with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c
/root/built_in_scripts/build_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc};
+fi
+
+echo "Built the bro plugin"
+
diff --git a/docker/scripts/docker_execute_configure_bro_plugin.sh
b/docker/scripts/docker_execute_configure_bro_plugin.sh
new file mode 100755
index 0000000..c4011c4
--- /dev/null
+++ b/docker/scripts/docker_execute_configure_bro_plugin.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the configure_bro_plugin.sh in the docker container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+CONTAINER_NAME=bro
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_execute_configure_bro_plugin with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c
/root/built_in_scripts/configure_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc};
+fi
+
+echo "configured the bro kafka plugin"
+
diff --git a/docker/scripts/docker_execute_process_data_file.sh
b/docker/scripts/docker_execute_process_data_file.sh
new file mode 100755
index 0000000..a944fb2
--- /dev/null
+++ b/docker/scripts/docker_execute_process_data_file.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the process_data_dir.sh script in the container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " --pcap-file-name [REQUIRED] The name of the pcap
file"
+ echo " --output-directory-name [REQUIRED] The name of the output
directory"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+CONTAINER_NAME=bro
+PCAP_FILE_NAME=
+OUTPUT_DIRECTORY_NAME=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # PCAP_FILE_NAME
+ #
+ # --pcap-file-name
+ #
+ --pcap-file-name=*)
+ PCAP_FILE_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # OUTPUT_DIRECTORY_NAME
+ #
+ # --output-directory-name
+ #
+ --output-directory-name=*)
+ OUTPUT_DIRECTORY_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_execute_process_data_dir with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "PCAP_FILE_NAME = ${PCAP_FILE_NAME}"
+echo "OUTPUT_DIRECTORY_NAME = ${OUTPUT_DIRECTORY_NAME}"
+echo "==================================================="
+
+echo "executing process_data_file.sh in the bro docker container"
+echo " "
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c
"built_in_scripts/process_data_file.sh --pcap-file-name=${PCAP_FILE_NAME}
--output-directory-name=${OUTPUT_DIRECTORY_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc};
+fi
+
+echo "done processing ${PCAP_FILE_NAME}"
\ No newline at end of file
diff --git a/docker/scripts/docker_execute_shell.sh
b/docker/scripts/docker_execute_shell.sh
new file mode 100755
index 0000000..ff17a1a
--- /dev/null
+++ b/docker/scripts/docker_execute_shell.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Gets a bash shell for a container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+CONTAINER_NAME=bro
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running build_bro_plugin_docker with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root -i -t "${CONTAINER_NAME}" bash
+
diff --git a/docker/scripts/docker_run_bro_container.sh
b/docker/scripts/docker_run_bro_container.sh
new file mode 100755
index 0000000..1e31b25
--- /dev/null
+++ b/docker/scripts/docker_run_bro_container.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the bro container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: bro"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " --scripts-path [OPTIONAL] The path with the
scripts you may run in the container. These are your scripts, not the built in
scripts"
+ echo " --data-path [OPTIONAL] The name of the
directory to map to /root/data in the container"
+ echo " --test-output-path [REQUIRED] The path to log test
data to"
+ echo " --docker-parameter [OPTIONAL, MULTIPLE] Each
parameter with this name will be passed to docker run"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+BRO_PLUGIN_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd ../..
&& pwd)"
+CONTAINER_NAME=bro
+NETWORK_NAME=bro-network
+OUR_SCRIPTS_PATH="${BRO_PLUGIN_PATH}/docker/in_docker_scripts"
+SCRIPTS_PATH=
+DATA_PATH=
+TEST_OUTPUT_PATH=
+
+declare -a DOCKER_PARAMETERS
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # DATA_PATH
+ #
+ # --data-path
+ #
+ --data-path=*)
+ DATA_PATH="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # SCRIPTS_PATH
+ #
+ # --scripts-path
+ #
+ --scripts-path=*)
+ SCRIPTS_PATH="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # TEST_OUTPUT_PATH
+ #
+ # --test-output-path
+ #
+ --test-output-path=*)
+ TEST_OUTPUT_PATH="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # DOCKER_PARAMETERS
+ #
+ # --docker-parameter
+ #
+ --docker-parameter=*)
+ DOCKER_PARAMETERS=( "${DOCKER_PARAMETERS[@]}" "${i#*=}" )
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+ esac
+done
+
+echo "Running docker_run_bro_container with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "NETWORK_NAME = ${NETWORK_NAME}"
+echo "SCRIPT_PATH = ${SCRIPTS_PATH}"
+echo "DATA_PATH = ${DATA_PATH}"
+echo "TEST_OUTPUT_PATH = ${TEST_OUTPUT_PATH}"
+echo "DOCKER_PARAMETERS = " "${DOCKER_PARAMETERS[@]}"
+echo "==================================================="
+
+
+# Build the docker command line
+declare -a DOCKER_CMD_BASE
+DOCKER_CMD="bash"
+DOCKER_CMD_BASE[0]="docker run -d -t --name ${CONTAINER_NAME} --network
${NETWORK_NAME} "
+DOCKER_CMD_BASE[2]="-v \"${OUR_SCRIPTS_PATH}:/root/built_in_scripts\" "
+DOCKER_CMD_BASE[3]="-v \"${BRO_PLUGIN_PATH}:/root/code\" "
+DOCKER_CMD_BASE[4]="-v \"${TEST_OUTPUT_PATH}:/root/test_output\" "
+OFFSET=5
+if [[ -n "$SCRIPTS_PATH" ]]; then
+ DOCKER_CMD_BASE[$OFFSET]="-v \"${SCRIPTS_PATH}:/root/scripts\" "
+ OFFSET=6
+fi
+
+if [[ -n "$DATA_PATH" ]]; then
+ DOCKER_CMD_BASE[$OFFSET]="-v \"${DATA_PATH}:/root/data\" "
+fi
+
+echo "===============Running Docker==============="
+echo ""
+echo "eval command is: "
+echo "${DOCKER_CMD_BASE[@]}" "${DOCKER_PARAMETERS[@]}" "${CONTAINER_NAME}"
"${DOCKER_CMD}"
+echo ""
+echo "============================================"
+echo ""
+eval "${DOCKER_CMD_BASE[@]}" "${DOCKER_PARAMETERS[@]}"
metron-bro-docker-container:latest "${DOCKER_CMD}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+echo "Started bro container"
+echo " "
+echo " "
+
diff --git a/docker/scripts/docker_run_consume_bro_kafka.sh
b/docker/scripts/docker_run_consume_bro_kafka.sh
new file mode 100755
index 0000000..eea740a
--- /dev/null
+++ b/docker/scripts/docker_run_consume_bro_kafka.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs a kafka container with the console consumer for the bro topic. The
consumer should quit when it has read
+# all of the messages available
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " --offset [OPTIONAL] The kafka offset to
read from. Default: -1"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+OFFSET=-1
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # OFFSET
+ #
+ # --offset
+ #
+ --offset=*)
+ OFFSET="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+ kafka-console-consumer.sh --topic bro --offset "${OFFSET}" --partition 0
--bootstrap-server kafka:9092 --timeout-ms 1000
+
diff --git a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
new file mode 100755
index 0000000..cb00e2c
--- /dev/null
+++ b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_run_create_bro_topic_in_kafka with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+ kafka-topics.sh --create --topic bro --replication-factor 1 --partitions 1
--zookeeper zookeeper:2181
+
diff --git a/docker/scripts/docker_run_get_offset_bro_kafka.sh
b/docker/scripts/docker_run_get_offset_bro_kafka.sh
new file mode 100755
index 0000000..d3b6621
--- /dev/null
+++ b/docker/scripts/docker_run_get_offset_bro_kafka.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs a kafka container with the console consumer for the bro topic. The
consumer should quit when it has read
+# all of the messages available
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+ kafka-run-class.sh kafka.tools.GetOffsetShell --topic bro --broker-list
kafka:9092
+
diff --git a/docker/scripts/docker_run_kafka_container.sh
b/docker/scripts/docker_run_kafka_container.sh
new file mode 100755
index 0000000..a7d6781
--- /dev/null
+++ b/docker/scripts/docker_run_kafka_container.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the kafka container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_run_kafka_container with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run -d --name kafka --network "${NETWORK_NAME}" --env
ZOOKEEPER_IP=zookeeper ches/kafka
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+echo "Started the kafka container with network ${NETWORK_NAME}"
+
diff --git a/docker/scripts/docker_run_wait_for_kafka.sh
b/docker/scripts/docker_run_wait_for_kafka.sh
new file mode 100755
index 0000000..3d659c9
--- /dev/null
+++ b/docker/scripts/docker_run_wait_for_kafka.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes a wait script for kafka
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+DOCKER_SCRIPTS_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd
.. > /dev/null && cd in_docker_scripts && pwd)"
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_run_wait_for_kakfa with"
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm -i -t -w /root --network "${NETWORK_NAME}" -v
"${DOCKER_SCRIPTS_PATH}":/root/scripts centos bash -c "bash
/root/scripts/wait_for_kafka.sh"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_run_wait_for_zookeeper.sh
b/docker/scripts/docker_run_wait_for_zookeeper.sh
new file mode 100755
index 0000000..5ce60ae
--- /dev/null
+++ b/docker/scripts/docker_run_wait_for_zookeeper.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes a wait script for zookeeper
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+DOCKER_SCRIPTS_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd
.. > /dev/null && cd in_docker_scripts && pwd)"
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_run_wait_for_zookeeper with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm -i -t -w /root --network "${NETWORK_NAME}" -v
"${DOCKER_SCRIPTS_PATH}":/root/scripts centos bash -c "bash
/root/scripts/wait_for_zk.sh"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_run_zookeeper_container.sh
b/docker/scripts/docker_run_zookeeper_container.sh
new file mode 100755
index 0000000..c6024b0
--- /dev/null
+++ b/docker/scripts/docker_run_zookeeper_container.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the zookeeper container
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # NETWORK_NAME
+ #
+ # --network-name
+ #
+ --network-name=*)
+ NETWORK_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+echo "Running docker_run_zookeeper_container with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run -d --name zookeeper --network "${NETWORK_NAME}" zookeeper:3.4
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+echo "Started the zookeeper container with networ ${NETWORK_NAME}"
+
diff --git a/docker/scripts/download_sample_pcaps.sh
b/docker/scripts/download_sample_pcaps.sh
new file mode 100755
index 0000000..6a12af3
--- /dev/null
+++ b/docker/scripts/download_sample_pcaps.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Downloads sample pcap files to the data directory
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --data-path [REQUIRED] The pcap data path"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+DATA_PATH=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # DATA_PATH
+ #
+ # --data-path
+ #
+ --data-path=*)
+ DATA_PATH="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "$DATA_PATH" ]]; then
+ echo "DATA_PATH must be passed"
+ exit 1
+fi
+
+echo "Running download_sample_pcaps with "
+echo "DATA_PATH = $DATA_PATH"
+echo "==================================================="
+
+for folder in nitroba example-traffic ssh ftp radius rfb; do
+ if [[ ! -d "${DATA_PATH}"/${folder} ]]; then
+ mkdir -p "${DATA_PATH}"/${folder}
+ fi
+done
+
+if [[ ! -f "${DATA_PATH}"/example-traffic/exercise-traffic.pcap ]]; then
+ wget https://www.bro.org/static/traces/exercise-traffic.pcap -O
"${DATA_PATH}"/example-traffic/exercise-traffic.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
+ wget
http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap
-O "${DATA_PATH}"/nitroba/nitroba.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/ssh/ssh.pcap ]]; then
+ wget https://www.bro.org/static/traces/ssh.pcap -O
"${DATA_PATH}"/ssh/ssh.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/ftp/ftp.pcap ]]; then
+ wget
https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true
-O "${DATA_PATH}"/ftp/ftp.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/radius/radius_localhost.pcapng ]]; then
+ wget
https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true
-O "${DATA_PATH}"/radius/radius_localhost.pcapng
+fi
+
+if [[ ! -f "${DATA_PATH}"/rfb/rfb.pcap ]]; then
+ wget
https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc-openwall-3.7.pcap?raw=true
-O "${DATA_PATH}"/rfb/rfb.pcap
+fi
+
diff --git a/docker/scripts/print_results.sh b/docker/scripts/print_results.sh
new file mode 100755
index 0000000..ecc67ca
--- /dev/null
+++ b/docker/scripts/print_results.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Prints all the results.csv files
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --test-directory [REQUIRED] The directory for the tests"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+TEST_DIRECTORY=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # TEST_DIRECTORY
+ #
+ # --test-directory
+ #
+ --test-directory=*)
+ TEST_DIRECTORY="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "$TEST_DIRECTORY" ]]; then
+ echo "$TEST_DIRECTORY must be passed"
+ exit 1
+fi
+
+
+echo "Running with "
+echo "TEST_DIRECTORY = $TEST_DIRECTORY"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${TEST_DIRECTORY}" || exit 1
+find "${TEST_DIRECTORY}" -name "results.csv" \
+ -exec echo "-->" '{}' \; \
+ -exec column -t -s ',' '{}' \; \
+ -exec echo "========================================================" \; \
+ -exec echo "" \;
+
diff --git a/docker/scripts/split_kakfa_output_by_log.sh
b/docker/scripts/split_kakfa_output_by_log.sh
new file mode 100755
index 0000000..71ec82b
--- /dev/null
+++ b/docker/scripts/split_kakfa_output_by_log.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2143,SC1083,SC2002,SC2126
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# For a given directory, finds all the bro log output, and splits the kafka
output file
+# by bro log, such that there is a bro log -> bro log kafka log
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --log-directory [REQUIRED] The directory with the
logs"
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+LOG_DIRECTORY=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # LOG_DIRECTORY
+ #
+ # --log-directory
+ #
+ --log-directory=*)
+ LOG_DIRECTORY="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "$LOG_DIRECTORY" ]]; then
+ echo "$LOG_DIRECTORY must be passed"
+ exit 1
+fi
+
+echo "Running with "
+echo "$LOG_DIRECTORY = $LOG_DIRECTORY"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${LOG_DIRECTORY}" || exit 1
+
+# for each log file, that is NOT KAFKA_OUTPUT_FILE we want to get the name
+# and extract the start
+# then we want to grep that name > name.kafka.log from the KAFKA_OUTPUT_FILE
+RESULTS_FILE="${LOG_DIRECTORY}/results.csv"
+echo "LOG,BRO_COUNT,KAFKA_COUNT" >> "${RESULTS_FILE}"
+for log in "${LOG_DIRECTORY}"/*.log
+do
+ BASE_LOG_FILE_NAME=$(basename "$log" .log)
+ if [[ ! "$BASE_LOG_FILE_NAME" == "kafka-output.log" ]]; then
+ if [[ $(grep {\""${BASE_LOG_FILE_NAME}"\":
"${LOG_DIRECTORY}"/kafka-output.log) ]]; then
+ grep {\""${BASE_LOG_FILE_NAME}"\": "${LOG_DIRECTORY}"/kafka-output.log >
"${LOG_DIRECTORY}"/"${BASE_LOG_FILE_NAME}".kafka.log
+
+ KAKFA_COUNT=$(cat "${LOG_DIRECTORY}/${BASE_LOG_FILE_NAME}.kafka.log" |
wc -l)
+ BRO_COUNT=$(grep -v "#" "${log}" | wc -l)
+
+ echo "${BASE_LOG_FILE_NAME},${BRO_COUNT},${KAKFA_COUNT}" >>
"${RESULTS_FILE}"
+ fi
+ fi
+done
+
diff --git a/docker/scripts/stop_container.sh b/docker/scripts/stop_container.sh
new file mode 100755
index 0000000..a26e2c7
--- /dev/null
+++ b/docker/scripts/stop_container.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+# Stops the Docker container with a given name
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --container-name [REQUIRED] The Docker container
name"
+ echo " -h/--help Usage information."
+ echo " "
+}
+
+CONTAINER_NAME=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # CONTAINER_NAME
+ #
+ # --container-name
+ #
+ --container-name=*)
+ CONTAINER_NAME="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+ echo "CONTAINER_NAME must be passed"
+ exit 1
+fi
+
+echo "Running stop_container with"
+echo "CONTAINER_NAME= $CONTAINER_NAME"
+echo "==================================================="
+
+docker stop "${CONTAINER_NAME}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ exit ${rc}
+fi
+
+docker rm "${CONTAINER_NAME}"
+
diff --git a/docker/test_output/.gitignore b/docker/test_output/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/docker/test_output/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
