This is an automated email from the ASF dual-hosted git repository. otto pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push: new c497078 METRON-2013 The bro plugin docker script's topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 c497078 is described below commit c49707874284dec11c37af138ffd99d503df9836 Author: JonZeolla <zeo...@gmail.com> AuthorDate: Thu Feb 28 15:48:32 2019 -0500 METRON-2013 The bro plugin docker script's topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 --- docker/README.md | 18 ++++++++------ docker/run_end_to_end.sh | 26 ++++++++++++++------ ...et_bro_kafka.sh => docker_run_consume_kafka.sh} | 28 +++++++++++++++++++--- ...afka.sh => docker_run_create_topic_in_kafka.sh} | 15 ++++++++++-- ...bro_kafka.sh => docker_run_get_offset_kafka.sh} | 18 +++++++------- 5 files changed, 77 insertions(+), 28 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3bae93b..a965d8b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -71,9 +71,9 @@ testing scripts to be added to a pull request, and subsequently to a test suite. ├── docker_execute_process_data_file.sh ├── docker_execute_shell.sh ├── docker_run_bro_container.sh -├── docker_run_consume_bro_kafka.sh -├── docker_run_create_bro_topic_in_kafka.sh -├── docker_run_get_offset_bro_topic_in_kafka.sh +├── docker_run_consume_kafka.sh +├── docker_run_create_topic_in_kafka.sh +├── docker_run_get_offset_kafka.sh ├── docker_run_kafka_container.sh ├── docker_run_wait_for_kafka.sh ├── docker_run_wait_for_zookeeper.sh @@ -146,22 +146,25 @@ testing scripts to be added to a pull request, and subsequently to a test suite. > You can then execute these scripts or use them together as part of testing etc. by creating `docker execute` scripts like those here. > The goal is to allow an individual to use and maintain their own library of scripts to use instead of, or in concert with the scripts maintained by this project. -- `docker_run_consume_bro_kafka.sh`: Runs an instance of the kafka container, with the console consumer `kafka-console-consumer.sh --topic bro --offset $OFFSET --partition 0 --bootstrap-server kafka:9092` +- `docker_run_consume_kafka.sh`: Runs an instance of the kafka container, with the console consumer `kafka-console-consumer.sh --topic $KAFKA_TOPIC --offset $OFFSET --partition 0 --bootstrap-server kafka:9092` ###### Parameters ```bash --network-name [OPTIONAL] The Docker network name. Default: bro-network --offset [OPTIONAL] The kafka offset. Default: -1 + --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro ``` -- `docker_run_get_offset_bro_kafka.sh`: Runs an instance of the kafka container and gets the current offset for the bro topic +- `docker_run_get_offset_kafka.sh`: Runs an instance of the kafka container and gets the current offset for the specified topic ###### Parameters ```bash --network-name [OPTIONAL] The Docker network name. Default: bro-network --offset [OPTIONAL] The kafka offset. Default: -1 + --kafka-topic [OPTIONAL] The kafka topic to get the offset from. Default: bro ``` -- `docker_run_create_bro_topic_in_kafka.sh`: Runs an instance of the kafka container, creating the `bro` topic +- `docker_run_create_topic_in_kafka.sh`: Runs an instance of the kafka container, creating the specified topic ###### Parameters ```bash --network-name [OPTIONAL] The Docker network name. Default: bro-network + --kafka-topic [OPTIONAL] The kafka topic to create. Default: bro ``` - `docker_run_kafka_container.sh`: Runs the main instance of the kafka container in the background ###### Parameters @@ -223,7 +226,7 @@ This script does the following: 3. Waits for zookeeper to be available 4. Runs the kafka container 5. Waits for kafka to be available -6. Creates the bro topic +6. Creates the specified topic 7. Downloads sample PCAP data 8. Runs the bro container in the background @@ -294,4 +297,5 @@ Other scripts may then be used to do your testing, for example running: ```bash --skip-docker-build [OPTIONAL] Skip build of bro docker machine. --data-path [OPTIONAL] The pcap data path. Default: ./data +--kafka-topic [OPTIONAL] The kafka topic name to use. Default: bro ``` diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh index a1dfb6a..6dfd146 100755 --- a/docker/run_end_to_end.sh +++ b/docker/run_end_to_end.sh @@ -28,6 +28,7 @@ function help { echo "USAGE" echo " --skip-docker-build [OPTIONAL] Skip build of bro docker machine." echo " --data-path [OPTIONAL] The pcap data path. Default: ./data" + echo " --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro" echo " -h/--help Usage information." echo " " echo "COMPATABILITY" @@ -51,10 +52,11 @@ DATA_PATH="${ROOT_DIR}"/data DATE=$(date) LOG_DATE=${DATE// /_} TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_} +KAFKA_TOPIC="bro" + # Handle command line options for i in "$@"; do case $i in - # # SKIP_REBUILD_BRO # @@ -64,7 +66,6 @@ for i in "$@"; do SKIP_REBUILD_BRO=true shift # past argument ;; - # # DATA_PATH # @@ -72,7 +73,15 @@ for i in "$@"; do DATA_PATH="${i#*=}" shift # past argument=value ;; - + # + # KAFKA_TOPIC + # + # --kafka-topic + # + --kafka-topic=*) + KAFKA_TOPIC="${i#*=}" + shift # past argument=value + ;; # # -h/--help # @@ -88,6 +97,8 @@ EXTRA_ARGS="$*" echo "Running build_container with " echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO" +echo "DATA_PATH = $DATA_PATH" +echo "KAFKA_TOPIC = $KAFKA_TOPIC" echo "===================================================" # Create the network @@ -120,8 +131,8 @@ rc=$?; if [[ ${rc} != 0 ]]; then exit ${rc} fi -# Create the bro topic -bash "${SCRIPT_DIR}"/docker_run_create_bro_topic_in_kafka.sh +# Create the kafka topic +bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic=${KAFKA_TOPIC} rc=$?; if [[ ${rc} != 0 ]]; then exit ${rc} fi @@ -187,7 +198,7 @@ do # get the current offset in kafka # this is where we are going to _start_ - OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_bro_kafka.sh | sed 's/^bro:0:\(.*\)$/\1/') + OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic=${KAFKA_TOPIC} | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/") echo "OFFSET------------------> ${OFFSET}" bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}" --output-directory-name="${DOCKER_DIRECTORY_NAME}" @@ -197,7 +208,8 @@ do fi KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log" - bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}" + bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset=${OFFSET} --kafka-topic=${KAFKA_TOPIC} | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}" + rc=$?; if [[ ${rc} != 0 ]]; then echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS" fi diff --git a/docker/scripts/docker_run_get_offset_bro_kafka.sh b/docker/scripts/docker_run_consume_kafka.sh similarity index 69% rename from docker/scripts/docker_run_get_offset_bro_kafka.sh rename to docker/scripts/docker_run_consume_kafka.sh index eafd08f..71e7cf9 100755 --- a/docker/scripts/docker_run_get_offset_bro_kafka.sh +++ b/docker/scripts/docker_run_consume_kafka.sh @@ -24,19 +24,23 @@ set -E # errtrap set -o pipefail # -# Runs a kafka container with the console consumer for the bro topic. The consumer should quit when it has read -# all of the messages available +# Runs a kafka container with the console consumer for the appropriate topic. +# The consumer should quit when it has read all of the messages available. # function help { echo " " echo "usage: ${0}" echo " --network-name [OPTIONAL] The Docker network name. Default: bro-network" + echo " --offset [OPTIONAL] The kafka offset to read from. Default: -1" + echo " --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro" echo " -h/--help Usage information." echo " " } NETWORK_NAME=bro-network +OFFSET=-1 +KAFKA_TOPIC=bro # handle command line options for i in "$@"; do @@ -51,6 +55,24 @@ for i in "$@"; do shift # past argument=value ;; # + # OFFSET + # + # --offset + # + --offset=*) + OFFSET="${i#*=}" + shift # past argument=value + ;; + # + # KAFKA_TOPIC + # + # --kafka-topic + # + --kafka-topic=*) + KAFKA_TOPIC="${i#*=}" + shift # past argument=value + ;; + # # -h/--help # -h | --help) @@ -71,7 +93,7 @@ for i in "$@"; do done docker run --rm --network "${NETWORK_NAME}" ches/kafka \ - kafka-run-class.sh kafka.tools.GetOffsetShell --topic bro --broker-list kafka:9092 + kafka-console-consumer.sh --topic "${KAFKA_TOPIC}" --offset "${OFFSET}" --partition 0 --bootstrap-server kafka:9092 --timeout-ms 1000 rc=$?; if [[ ${rc} != 0 ]]; then exit ${rc} fi diff --git a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh b/docker/scripts/docker_run_create_topic_in_kafka.sh similarity index 81% rename from docker/scripts/docker_run_create_bro_topic_in_kafka.sh rename to docker/scripts/docker_run_create_topic_in_kafka.sh index f4c84ab..f140469 100755 --- a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh +++ b/docker/scripts/docker_run_create_topic_in_kafka.sh @@ -27,11 +27,13 @@ function help { echo " " echo "usage: ${0}" echo " --network-name [OPTIONAL] The Docker network name. Default: bro-network" + echo " --kafka-topic [OPTIONAL] The kafka topic to create. Default: bro" echo " -h/--help Usage information." echo " " } NETWORK_NAME=bro-network +KAFKA_TOPIC=bro # handle command line options for i in "$@"; do @@ -46,6 +48,15 @@ for i in "$@"; do shift # past argument=value ;; # + # KAFKA_TOPIC + # + # --kafka-topic + # + --kafka-topic=*) + KAFKA_TOPIC="${i#*=}" + shift # past argument=value + ;; + # # -h/--help # -h | --help) @@ -65,12 +76,12 @@ for i in "$@"; do esac done -echo "Running docker_run_create_bro_topic_in_kafka with " +echo "Running docker_run_create_topic_in_kafka with " echo "NETWORK_NAME = $NETWORK_NAME" echo "===================================================" docker run --rm --network "${NETWORK_NAME}" ches/kafka \ - kafka-topics.sh --create --topic bro --replication-factor 1 --partitions 1 --zookeeper zookeeper:2181 + kafka-topics.sh --create --topic "${KAFKA_TOPIC}" --replication-factor 1 --partitions 1 --zookeeper zookeeper:2181 rc=$?; if [[ ${rc} != 0 ]]; then exit ${rc} fi diff --git a/docker/scripts/docker_run_consume_bro_kafka.sh b/docker/scripts/docker_run_get_offset_kafka.sh similarity index 79% rename from docker/scripts/docker_run_consume_bro_kafka.sh rename to docker/scripts/docker_run_get_offset_kafka.sh index 5a7bbcf..7d658fc 100755 --- a/docker/scripts/docker_run_consume_bro_kafka.sh +++ b/docker/scripts/docker_run_get_offset_kafka.sh @@ -24,21 +24,21 @@ set -E # errtrap set -o pipefail # -# Runs a kafka container with the console consumer for the bro topic. The consumer should quit when it has read -# all of the messages available +# Runs a kafka container with the console consumer for the provided topic. The +# consumer should quit when it has read all of the messages available. # function help { echo " " echo "usage: ${0}" echo " --network-name [OPTIONAL] The Docker network name. Default: bro-network" - echo " --offset [OPTIONAL] The kafka offset to read from. Default: -1" + echo " --kafka-topic [OPTIONAL] The kafka topic to pull the offset from. Default: bro" echo " -h/--help Usage information." echo " " } NETWORK_NAME=bro-network -OFFSET=-1 +KAFKA_TOPIC=bro # handle command line options for i in "$@"; do @@ -53,12 +53,12 @@ for i in "$@"; do shift # past argument=value ;; # - # OFFSET + # KAFKA_TOPIC # - # --offset + # --kafka-topic # - --offset=*) - OFFSET="${i#*=}" + --kafka-topic=*) + KAFKA_TOPIC="${i#*=}" shift # past argument=value ;; # @@ -82,7 +82,7 @@ for i in "$@"; do done docker run --rm --network "${NETWORK_NAME}" ches/kafka \ - kafka-console-consumer.sh --topic bro --offset "${OFFSET}" --partition 0 --bootstrap-server kafka:9092 --timeout-ms 1000 + kafka-run-class.sh kafka.tools.GetOffsetShell --topic "${KAFKA_TOPIC}" --broker-list kafka:9092 rc=$?; if [[ ${rc} != 0 ]]; then exit ${rc} fi