metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [metron-bro-plugin-kafka] branch master updated: METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21
Date Wed, 06 Feb 2019 14:26:10 GMT
This is an automated email from the ASF dual-hosted git repository.

otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
     new 04593a6  METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21
04593a6 is described below

commit 04593a6a2d6ee4b3b1d43aab70ad4a406be240fc
Author: ottobackwards <ottobackwards@gmail.com>
AuthorDate: Wed Feb 6 09:25:51 2019 -0500

    METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21
---
 .gitignore                                         |  12 +
 NOTICE                                             |   4 +-
 docker/README.md                                   | 291 +++++++++++++++++++++
 .../containers/bro-localbuild-container/Dockerfile |  49 ++++
 docker/data/.gitignore                             |   4 +
 docker/finish_end_to_end.sh                        |  35 +++
 docker/in_docker_scripts/build_bro_plugin.sh       |  43 +++
 docker/in_docker_scripts/configure_bro_plugin.sh   |  41 +++
 docker/in_docker_scripts/process_data_file.sh      |  78 ++++++
 docker/in_docker_scripts/wait-for-it.sh            | 162 ++++++++++++
 docker/in_docker_scripts/wait_for_kafka.sh         |  35 +++
 docker/in_docker_scripts/wait_for_zk.sh            |  35 +++
 docker/remove_timeout_message.sh                   |  51 ++++
 docker/run_end_to_end.sh                           | 202 ++++++++++++++
 docker/scripts/build_container.sh                  | 111 ++++++++
 docker/scripts/cleanup_docker.sh                   |  91 +++++++
 docker/scripts/create_docker_network.sh            |  79 ++++++
 docker/scripts/destroy_docker_network.sh           |  76 ++++++
 docker/scripts/docker_execute_build_bro_plugin.sh  |  84 ++++++
 .../scripts/docker_execute_configure_bro_plugin.sh |  84 ++++++
 docker/scripts/docker_execute_process_data_file.sh | 113 ++++++++
 docker/scripts/docker_execute_shell.sh             |  79 ++++++
 docker/scripts/docker_run_bro_container.sh         | 168 ++++++++++++
 docker/scripts/docker_run_consume_bro_kafka.sh     |  86 ++++++
 .../docker_run_create_bro_topic_in_kafka.sh        |  74 ++++++
 docker/scripts/docker_run_get_offset_bro_kafka.sh  |  75 ++++++
 docker/scripts/docker_run_kafka_container.sh       |  82 ++++++
 docker/scripts/docker_run_wait_for_kafka.sh        |  82 ++++++
 docker/scripts/docker_run_wait_for_zookeeper.sh    |  82 ++++++
 docker/scripts/docker_run_zookeeper_container.sh   |  82 ++++++
 docker/scripts/download_sample_pcaps.sh            | 112 ++++++++
 docker/scripts/print_results.sh                    |  91 +++++++
 docker/scripts/split_kakfa_output_by_log.sh        | 106 ++++++++
 docker/scripts/stop_container.sh                   |  85 ++++++
 docker/test_output/.gitignore                      |   4 +
 35 files changed, 2887 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 28a8358..735a17e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,9 @@
+#ide stuff
+.idea
+*.iml
+*.iws
+.DS_Store
+
 .state
 build
 
@@ -29,3 +35,9 @@ build
 *.exe
 *.out
 *.app
+
+# Log files
+*.log
+# pcap files
+*.pcap
+*.pcapng
diff --git a/NOTICE b/NOTICE
index 9505518..7f966de 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,7 @@
    Apache Metron
-   Copyright 2015-2016 The Apache Software Foundation
+   Copyright 2015-2018 The Apache Software Foundation
 
    This product includes software developed at
    The Apache Software Foundation (http://www.apache.org/).
+
+   This product bundles wait-for-it.sh, which is available under a "MIT Software License" license.  For details, see https://github.com/vishnubob/wait-for-it
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000..8e4d3fa
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,291 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+## Docker support for testing metron-bro-plugin-kafka
+
+These scripts and containers provide support for building and testing Bro and the metron-bro-plugin-kafka using a number of Docker containers.
+The use of these scripts and containers allow an easier, automated workflow for testing new features, fixes, or regressions than before.
+One of the goals is for this to be extensible, such that new scripts can be introduced and run as well.  This will allow, for example, one or more
+testing scripts to be added to a pull request, and subsequently to a test suite.
+
+
+#### Directories
+
+```bash
+├── containers
+│   └── bro-localbuild-container
+├── data
+├── in_docker_scripts
+├── scripts
+└── test_output
+```
+- `containers`: The parent of all of the containers that this project defines.  We use several containers, not all of them ours.
+  - `bro-localbuild-container`: The docker container directory for our bro container, used for building bro, the librdkafka, and our plugin, as well as running bro.
+- `data`: The default path for pcap data to be used in tests.
+- `in_docker_scripts`: This directory is mapped to the bro docker container as /root/built_in_scripts.  These represent the library of scripts we provide to be run in the docker container.
+- `scripts`: These are the scripts that are run on the host for creating the docker bits, running containers, running or executing commands against containers ( such as executing one of the built_in_scripts ), and cleaning up resources.
+- `test_output`: Directory where the bro logs and kafka logs per test/pcap are stored.
+
+
+#### Scripts that execute _in_ the docker container
+
+```bash
+├── build_bro_plugin.sh
+├── configure_bro_plugin.sh
+├── process_data_file.sh
+├── wait-for-it.sh
+├── wait_for_kafka.sh
+└── wait_for_zk.sh
+```
+
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `configure_bro_plugin.sh`: Configures the plugin for the kafka container, and routes all traffic types.
+- `process_data_file.sh`: Runs `bro -r` on the passed file
+- `wait-for-it.sh`: Waits for a port to be open, so we know something is available.
+- `wait_for_kafka.sh`: Waits for the kafka to be available.
+- `wait_for_zk.sh`: Waits for zookeeper to be available.
+
+
+#### Scripts executed on the host to setup and interact with the docker containers
+
+```bash
+├── build_container.sh
+├── cleanup_docker.sh
+├── create_docker_network.sh
+├── destroy_docker_network.sh
+├── docker_execute_build_bro_plugin.sh
+├── docker_execute_configure_bro_plugin.sh
+├── docker_execute_process_data_file.sh
+├── docker_execute_shell.sh
+├── docker_run_bro_container.sh
+├── docker_run_consume_bro_kafka.sh
+├── docker_run_create_bro_topic_in_kafka.sh
+├── docker_run_get_offset_bro_topic_in_kafka.sh
+├── docker_run_kafka_container.sh
+├── docker_run_wait_for_kafka.sh
+├── docker_run_wait_for_zookeeper.sh
+├── docker_run_zookeeper_container.sh
+├── download_sample_pcaps.sh
+├── print_results.sh
+├── split_kakfa_output_by_log.sh
+└── stop_container.sh
+```
+
+- `build_container.sh`: Runs docker build in the passed directory, and names the results
+  ###### Parameters
+  ```bash
+   --container-directory          [REQUIRED] The directory with the Dockerfile
+   --container-name               [REQUIRED] The name to give the Docker container
+  ```
+- `cleanup_containers.sh`: Stops the containers and destroys the network 
+  ###### Parameters
+  ```bash
+  --container-name                [OPTIONAL] The Docker container name. Default: bro
+  --network-name                  [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `create_docker_network.sh`: Creates the Docker network that the containers will use
+  ###### Parameters
+  ```bash
+  --network-name                  [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `destroy_docker_network.sh`: Destroys a Docker network by calling `docker network rm`
+  ###### Parameters
+  ```bash
+   --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `docker_execute_build_bro_plugin.sh`: Executes `build_bro_plugin.sh` in the bro container
+  ###### Parameters
+  ```bash
+   --container-name               [OPTIONAL] The Docker container name. Default: bro
+  ```
+- `docker_execute_configure_bro_plugin.sh`: Executes `configure_bro_plugin.sh` in the bro container
+  ###### Parameters
+  ```bash
+  --container-name                [OPTIONAL] The Docker container name. Default: bro
+  ```
+- `docker_execute_process_data_dir.sh`: Executes `process_data_dir.sh` in the bro container
+  ###### Parameters
+   ```bash
+   --container-name               [OPTIONAL] The Docker container name. Default: bro
+   ```
+- `docker_execute_shell.sh`: `docker execute -i -t bash` to get a shell in a given container
+  ###### Parameters
+  ```bash
+  --container-name                [OPTIONAL] The Docker container name. Default: bro
+  ```
+- `docker_run_bro_container.sh`:  Runs the bro docker container in the background
+  ###### Parameters
+  ```bash
+  --container-name                [OPTIONAL] The Docker container name. Default: bro
+  --network-name                  [OPTIONAL] The Docker network name. Default: bro-network
+  --scripts-path                  [OPTIONAL] The path with the scripts you may run in the container. These are your scripts, not the built in scripts
+  --data-path                     [OPTIONAL] The name of the directory to map to /root/data in the container
+  --docker-parameter              [OPTIONAL, MULTIPLE] Each parameter with this name will be passed to docker run
+  ```
+  
+  > NOTE about `--scripts-path`
+  > The scripts path provided with be mapped into the bro container at `/root/scripts`.  This allows you to _inject_ your own scripts (not managed as part of this source project) into the container.
+  > You can then execute these scripts or use them together as part of testing etc. by creating `docker execute` scripts like those here.
+  > The goal is to allow an individual to use and maintain their own library of scripts to use instead of, or in concert with the scripts maintained by this project.
+  
+- `docker_run_consume_bro_kafka.sh`: Runs an instance of the kafka container, with the console consumer `kafka-console-consumer.sh --topic bro --offset $OFFSET --partition 0 --bootstrap-server kafka:9092`
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  --offset                       [OPTIONAL] The kafka offset. Default: -1
+  ```
+- `docker_run_get_offset_bro_kafka.sh`: Runs an instance of the kafka container and gets the current offset for the bro topic
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  --offset                       [OPTIONAL] The kafka offset. Default: -1
+  ```
+- `docker_run_create_bro_topic_in_kafka.sh`: Runs an instance of the kafka container, creating the `bro` topic
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `docker_run_kafka_container.sh`: Runs the main instance of the kafka container in the background
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `docker_run_wait_for_kafka.sh`: Runs the `wait_for_kafka.sh` in a base centos container
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `docker_run_wait_for_zookeeper.sh`: Runs the `wait_for_zk.sh` in a base centos container
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `docker_run_zookeeper_container.sh`: Runs the zookeeper container in the background
+  ###### Parameters
+  ```bash
+  --network-name                 [OPTIONAL] The Docker network name. Default: bro-network
+  ```
+- `download_sample_pcaps.sh`: Downloads the sample pcaps to a specified directory. If they exist, it is a no-op
+  
+   > The sample pcaps are:
+   >  -  https://www.bro.org/static/traces/exercise-traffic.pcap 
+   >  -  http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap 
+   >  -  https://www.bro.org/static/traces/ssh.pcap 
+   >  -  https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true 
+   >  -  https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true 
+   >  -  https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc
+
+  ###### Parameters
+  ```bash
+  --data-path                    [REQUIRED] The pcap data path
+  ```
+- `print_results.sh` : Prints the `results.csv` for all the pcaps processed in the given directory to console
+  ###### Parameters
+  ```bash
+  --test-directory               [REQUIRED] The directory for the tests
+  ```
+- `split_kafka_output_by_log.sh` : For a pcap result directory, will create a LOG.kafka.log for each LOG.log's entry in the kafka-output.log
+  ###### Parameters
+  ```bash
+  --log-directory                [REQUIRED] The directory with the logs
+  ```
+- `stop_container.sh`: Stops and removes a Docker container with a given name
+  ###### Parameters
+  ```bash
+  --container-name               [REQUIRED] The Docker container name
+  ```
+
+#### The example end to end test script
+
+`run_end_to_end.sh` is provided as an example of a testing script.  Specific or extended scripts can be created similar to this script to use the containers.
+This script does the following:
+
+1. Creates the Docker network
+2. Runs the zookeeper container
+3. Waits for zookeeper to be available
+4. Runs the kafka container
+5. Waits for kafka to be available
+6. Creates the bro topic
+7. Downloads sample PCAP data
+8. Runs the bro container in the background
+
+> Note that all parameters passed to this script are passed to the `docker_run_bro_container.sh` script
+
+9. Builds the bro plugin
+10. Configures the bro plugin
+11. Runs bro against all the pcap data, one at a time
+12. Executes a kafka client to read the data from bro for each pcap file
+13. Stores the output kafka messages and the bro logs into the test_output directory
+
+```bash
+>tree Tue_Jan__8_21_54_10_EST_2019
+Tue_Jan__8_21_54_10_EST_2019
+├── exercise-traffic_pcap
+│   ├── capture_loss.log
+│   ├── conn.log
+│   ├── dhcp.log
+│   ├── dns.log
+│   ├── files.log
+│   ├── http.log
+│   ├── kafka-output.log
+│   ├── known_certs.log
+│   ├── known_devices.log
+│   ├── loaded_scripts.log
+│   ├── notice.log
+│   ├── packet_filter.log
+│   ├── reporter.log
+│   ├── smtp.log
+│   ├── software.log
+│   ├── ssl.log
+│   ├── stats.log
+│   ├── weird.log
+│   └── x509.log
+├── ftp_pcap
+│   ├── capture_loss.log
+│   ├── conn.log
+│   ├── files.log
+│   ├── ftp.log
+│   ├── kafka-output.log
+│   ├── loaded_scripts.log
+│   ├── packet_filter.log
+│   ├── reporter.log
+│   ├── software.log
+│   └── stats.log
+```
+
+14. Creates a results.csv for each pcap that has the line counts of the kafka and the bro output for each log
+15. Prints all the results.csv to the screen
+
+As we can see, the output is a folder named for the test run time, with a sub folder per pcap, containing all the bro logs and the kafka_output.log.
+
+
+At this point the containers are up and running in the background.
+
+Other scripts may then be used to do your testing, for example running:
+```bash
+./scripts/docker_execute_shell.sh
+```
+
+> NOTE: If the scripts are run repeatedly, and there is no change in bro or the librdkafka, the line `./run_end_to_end.sh` can be replaced by `./run_end_to_end.sh --skip-docker-build`, which uses the `--skip-docker-build` flag to not rebuild the bro container, saving the time of rebuilding bro and librdkafka.
+
+> NOTE: After you are done, you must call the `finish_end_to_end.sh` script to cleanup.
+
+
+##### `run_end_to_end.sh`
+###### Parameters
+```bash
+--skip-docker-build            [OPTIONAL] Skip build of bro docker machine.
+--data-path                    [OPTIONAL] The pcap data path. Default: ./data
+```
diff --git a/docker/containers/bro-localbuild-container/Dockerfile b/docker/containers/bro-localbuild-container/Dockerfile
new file mode 100644
index 0000000..f7ad66f
--- /dev/null
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -0,0 +1,49 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+FROM centos:7
+WORKDIR /root
+
+# install development tools
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel openssl-devel python-devel swig zlib-devel perl cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi git jq
+
+# install bro
+RUN curl -L https://www.bro.org/downloads/bro-2.5.5.tar.gz | tar xvz
+WORKDIR bro-2.5.5/
+RUN ./configure
+RUN make
+RUN make install
+ENV PATH="${PATH}:/usr/local/bro/bin"
+
+# install pip
+RUN yum -y update
+RUN yum -y install epel-release
+RUN yum -y install python-pip
+RUN pip install --upgrade pip
+
+# install bro-pkg
+RUN pip install bro-pkg
+ENV PATH="${PATH}:/usr/bin"
+RUN bro-pkg autoconfig
+
+# install librdkafka
+RUN curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz | tar xvz
+WORKDIR librdkafka-0.11.5/
+RUN ./configure --enable-sasl
+RUN make
+RUN make install
+WORKDIR /root
diff --git a/docker/data/.gitignore b/docker/data/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/docker/data/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
diff --git a/docker/finish_end_to_end.sh b/docker/finish_end_to_end.sh
new file mode 100755
index 0000000..1c43d79
--- /dev/null
+++ b/docker/finish_end_to_end.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#
+# This script should be run _after_ run_end_to_end.sh when you are finished with your testing and the containers.
+# Do not run this if you plan on running docker_execute_shell.sh for example
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+# Stop all the containers and remote the networks
+bash "${ROOT_DIR}"/scripts/cleanup_docker.sh
+
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh b/docker/in_docker_scripts/build_bro_plugin.sh
new file mode 100755
index 0000000..46277bd
--- /dev/null
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Runs bro-pkg to build and install the plugin
+#
+
+cd /root || exit 1
+
+echo "================================"
+
+bro-pkg install code --force
+rc=$?; if [[ ${rc} != 0 ]]; then
+  echo "ERROR running bro-pkg install ${rc}"
+  exit ${rc}
+fi
+
+echo "================================"
+echo ""
+
+bro -N Apache::Kafka
+
+echo "================================"
+echo ""
+
diff --git a/docker/in_docker_scripts/configure_bro_plugin.sh b/docker/in_docker_scripts/configure_bro_plugin.sh
new file mode 100755
index 0000000..74551c2
--- /dev/null
+++ b/docker/in_docker_scripts/configure_bro_plugin.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Configures the bro kafka plugin
+# Configures the kafka broker
+# Configures the plugin for all the traffic types
+#
+
+echo "Configuring kafka plugin"
+{
+  echo "@load packages"
+  echo "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG, DPD::LOG, FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG, Weird::LOG, Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG, X509::LOG, Known::DEVICES_LOG, RFB::LOG, Stats::LOG, CaptureLoss::LOG, SIP::LOG);"
+  echo "redef Kafka::topic_name = \"bro\";"
+  echo "redef Kafka::tag_json = T;"
+  echo "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"kafka:9092\");"
+  echo "redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);"
+  echo "redef Known::cert_tracking = ALL_HOSTS;"
+  echo "redef Software::asset_tracking = ALL_HOSTS;"
+} >> /usr/local/bro/share/bro/site/local.bro
+
+sed -i '86 a @load policy/protocols/dhcp/known-devices-and-hostnames.bro' /usr/local/bro/share/bro/site/local.bro
+
diff --git a/docker/in_docker_scripts/process_data_file.sh b/docker/in_docker_scripts/process_data_file.sh
new file mode 100755
index 0000000..021c5fd
--- /dev/null
+++ b/docker/in_docker_scripts/process_data_file.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2010
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+PCAP_FILE_NAME=
+OUTPUT_DIRECTORY_NAME=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # PCAP_FILE_NAME
+  #
+  #   --pcap-file-name
+  #
+    --pcap-file-name=*)
+      PCAP_FILE_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # OUTPUT_DIRECTORY_NAME
+  #
+  #   --output-directory-name
+  #
+    --output-directory-name=*)
+      OUTPUT_DIRECTORY_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "PCAP_FILE_NAME = ${PCAP_FILE_NAME}"
+echo "OUTPUT_DIRECTORY_NAME = ${OUTPUT_DIRECTORY_NAME}"
+
+cd /root || exit 1
+echo "================================"
+if [ ! -d /root/data ]; then
+  echo "DATA_PATH has not been set and mapped"
+  exit 1
+fi
+cd /root/test_output/"${OUTPUT_DIRECTORY_NAME}" || exit 1
+find /root/data -type f -name "${PCAP_FILE_NAME}" -exec echo "processing" '{}' \; -exec bro -r '{}' /usr/local/bro/share/bro/site/local.bro -C \;
+echo "done with ${PCAP_FILE_NAME}"
diff --git a/docker/in_docker_scripts/wait-for-it.sh b/docker/in_docker_scripts/wait-for-it.sh
new file mode 100755
index 0000000..73abf05
--- /dev/null
+++ b/docker/in_docker_scripts/wait-for-it.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2064,SC2124,SC2206,SC2086
+#   Use this script to test if a given TCP host/port are available
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+        result=$?
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi
diff --git a/docker/in_docker_scripts/wait_for_kafka.sh b/docker/in_docker_scripts/wait_for_kafka.sh
new file mode 100755
index 0000000..56045f4
--- /dev/null
+++ b/docker/in_docker_scripts/wait_for_kafka.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Waits until kafka is available on the default port
+#
+
+cd /root || exit 1
+
+if [ ! -d /root/scripts ]; then
+  echo "DOCKER_SCRIPTS_PATH has not been set and mapped"
+  exit 1
+fi
+
+cd /root/scripts || exit 1
+bash ./wait-for-it.sh kafka:9092
+
diff --git a/docker/in_docker_scripts/wait_for_zk.sh b/docker/in_docker_scripts/wait_for_zk.sh
new file mode 100755
index 0000000..db8c1ff
--- /dev/null
+++ b/docker/in_docker_scripts/wait_for_zk.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+
+#
+# Waits until zookeeper is available on the default port
+#
+
+cd /root || exit 1
+
+if [ ! -d /root/scripts ]; then
+  echo "DOCKER_SCRIPTS_PATH has not been set and mapped"
+  exit 1
+fi
+
+cd /root/scripts || exit 1
+bash ./wait-for-it.sh zookeeper:2181
+
diff --git a/docker/remove_timeout_message.sh b/docker/remove_timeout_message.sh
new file mode 100755
index 0000000..dc34f2c
--- /dev/null
+++ b/docker/remove_timeout_message.sh
@@ -0,0 +1,51 @@
+#! /usr/bin/env bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#
+# Remove the exception text from piped input when we have purposefully timed
+# out reading kafka
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+LAST_CMD=
+SKIP_EXCEPTION_TEXT=false
+
+while read -r CMD; do
+    if [[ ${CMD} =~ ('ERROR Error processing message') ]]; then
+        LAST_CMD=${CMD}
+    elif [[ ${CMD} =~ ('kafka.consumer.ConsumerTimeoutException') ]]; then
+        SKIP_EXCEPTION_TEXT=true
+    elif [[ "$SKIP_EXCEPTION_TEXT" = true ]]; then
+        if [[ ! ${CMD} =~ (^at) ]]; then
+            echo "${CMD}"
+        fi
+    else
+        if [[ -n "$LAST_CMD" ]]; then
+            LAST_CMD=
+        fi
+        if [[ ! ${CMD} =~ (^--) ]]; then
+            echo "${CMD}"
+        fi
+    fi
+done
+
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
new file mode 100755
index 0000000..950dc60
--- /dev/null
+++ b/docker/run_end_to_end.sh
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --skip-docker-build             [OPTIONAL] Skip build of bro docker machine."
+  echo "    --data-path                     [OPTIONAL] The pcap data path. Default: ./data"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+SKIP_REBUILD_BRO=false
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+SCRIPT_DIR="${ROOT_DIR}"/scripts
+CONTAINER_DIR="${ROOT_DIR}"/containers/bro-localbuild-container
+DATA_PATH="${ROOT_DIR}"/data
+DATE=$(date)
+LOG_DATE=${DATE// /_}
+TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
+# Handle command line options
+for i in "$@"; do
+  case $i in
+
+  #
+  # SKIP_REBUILD_BRO
+  #
+  #   --skip-docker-build
+  #
+    --skip-docker-build)
+      SKIP_REBUILD_BRO=true
+      shift # past argument
+    ;;
+
+  #
+  # DATA_PATH
+  #
+    --data-path=*)
+      DATA_PATH="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+  esac
+done
+
+EXTRA_ARGS="$*"
+
+echo "Running build_container with "
+echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO"
+echo "==================================================="
+
+# Create the network
+bash "${SCRIPT_DIR}"/create_docker_network.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Run the zookeeper container
+bash "${SCRIPT_DIR}"/docker_run_zookeeper_container.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Wait for zookeeper to be up
+bash "${SCRIPT_DIR}"/docker_run_wait_for_zookeeper.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Run the kafka container
+bash "${SCRIPT_DIR}"/docker_run_kafka_container.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Wait for kafka to be up
+bash "${SCRIPT_DIR}"/docker_run_wait_for_kafka.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Create the bro topic
+bash "${SCRIPT_DIR}"/docker_run_create_bro_topic_in_kafka.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Build the bro container
+if [[ "$SKIP_REBUILD_BRO" = false ]]; then
+  bash "${SCRIPT_DIR}"/build_container.sh \
+   --container-directory="${CONTAINER_DIR}" \
+   --container-name=metron-bro-docker-container:latest
+
+  rc=$?; if [[ ${rc} != 0 ]]; then
+    exit ${rc}
+  fi
+fi
+
+# Download the pcaps
+bash "${SCRIPT_DIR}"/download_sample_pcaps.sh --data-path="${DATA_PATH}"
+
+mkdir "${TEST_OUTPUT_PATH}" || exit 1
+
+# Run the bro container and optionally the passed script _IN_ the container
+bash "${SCRIPT_DIR}"/docker_run_bro_container.sh \
+  --data-path="${DATA_PATH}" \
+  --test-output-path="${TEST_OUTPUT_PATH}" \
+  "$EXTRA_ARGS"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+# Build the bro plugin
+bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  echo "ERROR> FAILED TO BUILD PLUGIN.  CHECK LOGS  ${rc}"
+  exit ${rc}
+fi
+
+# Configure it the bro plugin
+bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  echo "ERROR> FAILED TO CONFIGURE PLUGIN.  CHECK LOGS  ${rc}"
+  exit ${rc}
+fi
+
+
+# for each pcap in the data directory, we want to
+# run bro then read the output from kafka
+# and output both of them to the same directory named
+# for the date/pcap
+
+
+for file in "${DATA_PATH}"/**/*.pcap*
+do
+  # get the file name
+  BASE_FILE_NAME=$(basename "${file}")
+  DOCKER_DIRECTORY_NAME=${BASE_FILE_NAME//\./_}
+
+  mkdir "${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" || exit 1
+  echo "MADE ${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+
+  # get the current offset in kafka
+  # this is where we are going to _start_
+  OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_bro_kafka.sh | sed 's/^bro:0:\(.*\)$/\1/')
+  echo "OFFSET------------------> ${OFFSET}"
+
+  bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}" --output-directory-name="${DOCKER_DIRECTORY_NAME}"
+
+  rc=$?; if [[ ${rc} != 0 ]]; then
+    echo "ERROR> FAILED TO PROCESS ${file} DATA.  CHECK LOGS, please run the finish_end_to_end.sh when you are done."
+    exit ${rc}
+  fi
+  KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
+  bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
+
+  rc=$?; if [[ ${rc} != 0 ]]; then
+    echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA.  CHECK LOGS"
+  fi
+
+  "${SCRIPT_DIR}"/split_kakfa_output_by_log.sh --log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+done
+
+"${SCRIPT_DIR}"/print_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+
+echo ""
+echo "Run complete"
+echo "The kafka and bro output can be found at ${TEST_OUTPUT_PATH}"
+echo "You may now work with the containers if you will.  You need to call finish_end_to_end.sh when you are done"
diff --git a/docker/scripts/build_container.sh b/docker/scripts/build_container.sh
new file mode 100755
index 0000000..d4e5dca
--- /dev/null
+++ b/docker/scripts/build_container.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+#
+# Runs docker build in a provided directory, with a provided name
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-directory           [REQUIRED] The directory with the Dockerfile"
+  echo "    --container-name                [REQUIRED] The name to give the Docker container"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+CONTAINER_DIRECTORY=
+CONTAINER_NAME=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_DIRECTORY
+  #
+  #   --container-directory
+  #
+    --container-directory=*)
+      CONTAINER_DIRECTORY="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$CONTAINER_DIRECTORY" ]]; then
+  echo "CONTAINER_DIRECTORY must be passed"
+  exit 1
+fi
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+  echo "CONTAINER_NAME must be passed"
+  exit 1
+fi
+
+echo "Running with "
+echo "CONTAINER_DIRECTORY = $CONTAINER_DIRECTORY"
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${CONTAINER_DIRECTORY}" || exit 1
+echo "==================================================="
+echo "docker build of ${CONTAINER_NAME}"
+echo "==================================================="
+
+docker build . --no-cache --tag="${CONTAINER_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
diff --git a/docker/scripts/cleanup_docker.sh b/docker/scripts/cleanup_docker.sh
new file mode 100755
index 0000000..5a95e3a
--- /dev/null
+++ b/docker/scripts/cleanup_docker.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+#
+# Stops the containers, and shuts down the NETWORK_NAME
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+CONTAINER_NAME=bro
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument
+    ;;
+
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+  esac
+done
+
+echo "Running cleanup_containers with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "NETWORK_NAME   = $NETWORK_NAME"
+echo "==================================================="
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name="${CONTAINER_NAME}"
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name=kafka
+
+"${SCRIPT_DIR}"/stop_container.sh --container-name=zookeeper
+
+"${SCRIPT_DIR}"/destroy_docker_network.sh --network-name="${NETWORK_NAME}"
+
diff --git a/docker/scripts/create_docker_network.sh b/docker/scripts/create_docker_network.sh
new file mode 100755
index 0000000..efa1684
--- /dev/null
+++ b/docker/scripts/create_docker_network.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running create_docker_network with "
+echo "NETWORK_NAME   = $NETWORK_NAME"
+echo "==================================================="
+
+docker network create "${NETWORK_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+echo "Created network ${NETWORK_NAME}"
+
diff --git a/docker/scripts/destroy_docker_network.sh b/docker/scripts/destroy_docker_network.sh
new file mode 100755
index 0000000..516c33c
--- /dev/null
+++ b/docker/scripts/destroy_docker_network.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information"
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running destroy_docker_network with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker network rm "${NETWORK_NAME}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_execute_build_bro_plugin.sh b/docker/scripts/docker_execute_build_bro_plugin.sh
new file mode 100755
index 0000000..649fd4e
--- /dev/null
+++ b/docker/scripts/docker_execute_build_bro_plugin.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the build_bro_plugin.sh script in the container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+CONTAINER_NAME=bro
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running build_bro_plugin_docker with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c /root/built_in_scripts/build_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc};
+fi
+
+echo "Built the bro plugin"
+
diff --git a/docker/scripts/docker_execute_configure_bro_plugin.sh b/docker/scripts/docker_execute_configure_bro_plugin.sh
new file mode 100755
index 0000000..c4011c4
--- /dev/null
+++ b/docker/scripts/docker_execute_configure_bro_plugin.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the configure_bro_plugin.sh in the docker container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+CONTAINER_NAME=bro
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_execute_configure_bro_plugin with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c /root/built_in_scripts/configure_bro_plugin.sh
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc};
+fi
+
+echo "configured the bro kafka plugin"
+
diff --git a/docker/scripts/docker_execute_process_data_file.sh b/docker/scripts/docker_execute_process_data_file.sh
new file mode 100755
index 0000000..a944fb2
--- /dev/null
+++ b/docker/scripts/docker_execute_process_data_file.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes the process_data_dir.sh script in the container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    --pcap-file-name                [REQUIRED] The name of the pcap file"
+  echo "    --output-directory-name         [REQUIRED] The name of the output directory"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+CONTAINER_NAME=bro
+PCAP_FILE_NAME=
+OUTPUT_DIRECTORY_NAME=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # PCAP_FILE_NAME
+  #
+  #   --pcap-file-name
+  #
+    --pcap-file-name=*)
+      PCAP_FILE_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # OUTPUT_DIRECTORY_NAME
+  #
+  #   --output-directory-name
+  #
+    --output-directory-name=*)
+      OUTPUT_DIRECTORY_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_execute_process_data_dir with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "PCAP_FILE_NAME = ${PCAP_FILE_NAME}"
+echo "OUTPUT_DIRECTORY_NAME = ${OUTPUT_DIRECTORY_NAME}"
+echo "==================================================="
+
+echo "executing process_data_file.sh in the bro docker container"
+echo " "
+
+docker exec -w /root "${CONTAINER_NAME}" bash -c "built_in_scripts/process_data_file.sh --pcap-file-name=${PCAP_FILE_NAME} --output-directory-name=${OUTPUT_DIRECTORY_NAME}"
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc};
+fi
+
+echo "done processing ${PCAP_FILE_NAME}"
\ No newline at end of file
diff --git a/docker/scripts/docker_execute_shell.sh b/docker/scripts/docker_execute_shell.sh
new file mode 100755
index 0000000..ff17a1a
--- /dev/null
+++ b/docker/scripts/docker_execute_shell.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Gets a bash shell for a container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+CONTAINER_NAME=bro
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running build_bro_plugin_docker with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "==================================================="
+
+docker exec -w /root -i -t "${CONTAINER_NAME}" bash
+
diff --git a/docker/scripts/docker_run_bro_container.sh b/docker/scripts/docker_run_bro_container.sh
new file mode 100755
index 0000000..1e31b25
--- /dev/null
+++ b/docker/scripts/docker_run_bro_container.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the bro container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [OPTIONAL] The Docker container name. Default: bro"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    --scripts-path                  [OPTIONAL] The path with the scripts you may run in the container. These are your scripts, not the built in scripts"
+  echo "    --data-path                     [OPTIONAL] The name of the directory to map to /root/data in the container"
+  echo "    --test-output-path              [REQUIRED] The path to log test data to"
+  echo "    --docker-parameter              [OPTIONAL, MULTIPLE] Each parameter with this name will be passed to docker run"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+BRO_PLUGIN_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd ../.. && pwd)"
+CONTAINER_NAME=bro
+NETWORK_NAME=bro-network
+OUR_SCRIPTS_PATH="${BRO_PLUGIN_PATH}/docker/in_docker_scripts"
+SCRIPTS_PATH=
+DATA_PATH=
+TEST_OUTPUT_PATH=
+
+declare -a DOCKER_PARAMETERS
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # DATA_PATH
+  #
+  #   --data-path
+  #
+    --data-path=*)
+      DATA_PATH="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # SCRIPTS_PATH
+  #
+  #   --scripts-path
+  #
+    --scripts-path=*)
+      SCRIPTS_PATH="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # TEST_OUTPUT_PATH
+  #
+  #   --test-output-path
+  #
+    --test-output-path=*)
+      TEST_OUTPUT_PATH="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # DOCKER_PARAMETERS
+  #
+  #   --docker-parameter
+  #
+    --docker-parameter=*)
+      DOCKER_PARAMETERS=( "${DOCKER_PARAMETERS[@]}" "${i#*=}" )
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+  esac
+done
+
+echo "Running docker_run_bro_container with "
+echo "CONTAINER_NAME = $CONTAINER_NAME"
+echo "NETWORK_NAME = ${NETWORK_NAME}"
+echo "SCRIPT_PATH = ${SCRIPTS_PATH}"
+echo "DATA_PATH = ${DATA_PATH}"
+echo "TEST_OUTPUT_PATH = ${TEST_OUTPUT_PATH}"
+echo "DOCKER_PARAMETERS = " "${DOCKER_PARAMETERS[@]}"
+echo "==================================================="
+
+
+# Build the docker command line
+declare -a DOCKER_CMD_BASE
+DOCKER_CMD="bash"
+DOCKER_CMD_BASE[0]="docker run -d -t --name ${CONTAINER_NAME} --network ${NETWORK_NAME} "
+DOCKER_CMD_BASE[2]="-v \"${OUR_SCRIPTS_PATH}:/root/built_in_scripts\" "
+DOCKER_CMD_BASE[3]="-v \"${BRO_PLUGIN_PATH}:/root/code\" "
+DOCKER_CMD_BASE[4]="-v \"${TEST_OUTPUT_PATH}:/root/test_output\" "
+OFFSET=5
+if [[ -n "$SCRIPTS_PATH" ]]; then
+  DOCKER_CMD_BASE[$OFFSET]="-v \"${SCRIPTS_PATH}:/root/scripts\" "
+  OFFSET=6
+fi
+
+if [[ -n "$DATA_PATH" ]]; then
+  DOCKER_CMD_BASE[$OFFSET]="-v \"${DATA_PATH}:/root/data\" "
+fi
+
+echo "===============Running Docker==============="
+echo ""
+echo "eval command is: "
+echo "${DOCKER_CMD_BASE[@]}" "${DOCKER_PARAMETERS[@]}" "${CONTAINER_NAME}" "${DOCKER_CMD}"
+echo ""
+echo "============================================"
+echo ""
+eval "${DOCKER_CMD_BASE[@]}" "${DOCKER_PARAMETERS[@]}" metron-bro-docker-container:latest "${DOCKER_CMD}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+echo "Started bro container"
+echo " "
+echo " "
+
diff --git a/docker/scripts/docker_run_consume_bro_kafka.sh b/docker/scripts/docker_run_consume_bro_kafka.sh
new file mode 100755
index 0000000..eea740a
--- /dev/null
+++ b/docker/scripts/docker_run_consume_bro_kafka.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs a kafka container with the console consumer for the bro topic.  The consumer should quit when it has read
+# all of the messages available
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    --offset                        [OPTIONAL] The kafka offset to read from. Default: -1"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+OFFSET=-1
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # OFFSET
+  #
+  #   --offset
+  #
+    --offset=*)
+      OFFSET="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+  kafka-console-consumer.sh --topic bro --offset "${OFFSET}" --partition 0 --bootstrap-server kafka:9092 --timeout-ms 1000
+
diff --git a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
new file mode 100755
index 0000000..cb00e2c
--- /dev/null
+++ b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name.  Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_run_create_bro_topic_in_kafka with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+  kafka-topics.sh --create --topic bro --replication-factor 1 --partitions 1 --zookeeper zookeeper:2181
+
diff --git a/docker/scripts/docker_run_get_offset_bro_kafka.sh b/docker/scripts/docker_run_get_offset_bro_kafka.sh
new file mode 100755
index 0000000..d3b6621
--- /dev/null
+++ b/docker/scripts/docker_run_get_offset_bro_kafka.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs a kafka container with the console consumer for the bro topic.  The consumer should quit when it has read
+# all of the messages available
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+docker run --rm --network "${NETWORK_NAME}" ches/kafka \
+  kafka-run-class.sh kafka.tools.GetOffsetShell --topic bro --broker-list kafka:9092
+
diff --git a/docker/scripts/docker_run_kafka_container.sh b/docker/scripts/docker_run_kafka_container.sh
new file mode 100755
index 0000000..a7d6781
--- /dev/null
+++ b/docker/scripts/docker_run_kafka_container.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the kafka container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_run_kafka_container with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run -d --name kafka --network "${NETWORK_NAME}" --env ZOOKEEPER_IP=zookeeper ches/kafka
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+echo "Started the kafka container with network ${NETWORK_NAME}"
+
diff --git a/docker/scripts/docker_run_wait_for_kafka.sh b/docker/scripts/docker_run_wait_for_kafka.sh
new file mode 100755
index 0000000..3d659c9
--- /dev/null
+++ b/docker/scripts/docker_run_wait_for_kafka.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes a wait script for kafka
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+DOCKER_SCRIPTS_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd  .. > /dev/null && cd in_docker_scripts && pwd)"
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_run_wait_for_kakfa with"
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm -i -t -w /root --network "${NETWORK_NAME}" -v "${DOCKER_SCRIPTS_PATH}":/root/scripts centos bash -c "bash /root/scripts/wait_for_kafka.sh"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_run_wait_for_zookeeper.sh b/docker/scripts/docker_run_wait_for_zookeeper.sh
new file mode 100755
index 0000000..5ce60ae
--- /dev/null
+++ b/docker/scripts/docker_run_wait_for_zookeeper.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Executes a wait script for zookeeper
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+DOCKER_SCRIPTS_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && cd  .. > /dev/null && cd in_docker_scripts && pwd)"
+
+NETWORK_NAME=bro-network
+
+# handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_run_wait_for_zookeeper with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run --rm -i -t -w /root --network "${NETWORK_NAME}" -v "${DOCKER_SCRIPTS_PATH}":/root/scripts centos bash -c "bash /root/scripts/wait_for_zk.sh"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
diff --git a/docker/scripts/docker_run_zookeeper_container.sh b/docker/scripts/docker_run_zookeeper_container.sh
new file mode 100755
index 0000000..c6024b0
--- /dev/null
+++ b/docker/scripts/docker_run_zookeeper_container.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Runs the zookeeper container
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --network-name                  [OPTIONAL] The Docker network name. Default: bro-network"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+NETWORK_NAME=bro-network
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # NETWORK_NAME
+  #
+  #   --network-name
+  #
+    --network-name=*)
+      NETWORK_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+echo "Running docker_run_zookeeper_container with "
+echo "NETWORK_NAME = $NETWORK_NAME"
+echo "==================================================="
+
+docker run -d --name zookeeper --network "${NETWORK_NAME}" zookeeper:3.4
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+echo "Started the zookeeper container with networ ${NETWORK_NAME}"
+
diff --git a/docker/scripts/download_sample_pcaps.sh b/docker/scripts/download_sample_pcaps.sh
new file mode 100755
index 0000000..6a12af3
--- /dev/null
+++ b/docker/scripts/download_sample_pcaps.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Downloads sample pcap files to the data directory
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --data-path                    [REQUIRED] The pcap data path"
+  echo "    -h/--help                      Usage information."
+  echo " "
+  echo " "
+}
+
+DATA_PATH=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # DATA_PATH
+  #
+  #   --data-path
+  #
+    --data-path=*)
+      DATA_PATH="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$DATA_PATH" ]]; then
+  echo "DATA_PATH must be passed"
+  exit 1
+fi
+
+echo "Running download_sample_pcaps with "
+echo "DATA_PATH = $DATA_PATH"
+echo "==================================================="
+
+for folder in nitroba example-traffic ssh ftp radius rfb; do
+  if [[ ! -d "${DATA_PATH}"/${folder} ]]; then
+    mkdir -p "${DATA_PATH}"/${folder}
+  fi
+done
+
+if [[ ! -f "${DATA_PATH}"/example-traffic/exercise-traffic.pcap ]]; then
+  wget https://www.bro.org/static/traces/exercise-traffic.pcap -O "${DATA_PATH}"/example-traffic/exercise-traffic.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
+  wget http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap -O "${DATA_PATH}"/nitroba/nitroba.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/ssh/ssh.pcap ]]; then
+  wget https://www.bro.org/static/traces/ssh.pcap -O "${DATA_PATH}"/ssh/ssh.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/ftp/ftp.pcap ]]; then
+  wget https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true -O "${DATA_PATH}"/ftp/ftp.pcap
+fi
+
+if [[ ! -f "${DATA_PATH}"/radius/radius_localhost.pcapng ]]; then
+  wget https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true -O "${DATA_PATH}"/radius/radius_localhost.pcapng
+fi
+
+if [[ ! -f "${DATA_PATH}"/rfb/rfb.pcap ]]; then
+  wget https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc-openwall-3.7.pcap?raw=true -O "${DATA_PATH}"/rfb/rfb.pcap
+fi
+
diff --git a/docker/scripts/print_results.sh b/docker/scripts/print_results.sh
new file mode 100755
index 0000000..ecc67ca
--- /dev/null
+++ b/docker/scripts/print_results.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Prints all the results.csv files
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --test-directory           [REQUIRED] The directory for the tests"
+  echo "    -h/--help                   Usage information."
+  echo " "
+  echo " "
+}
+
+TEST_DIRECTORY=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # TEST_DIRECTORY
+  #
+  #   --test-directory
+  #
+    --test-directory=*)
+      TEST_DIRECTORY="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$TEST_DIRECTORY" ]]; then
+  echo "$TEST_DIRECTORY must be passed"
+  exit 1
+fi
+
+
+echo "Running with "
+echo "TEST_DIRECTORY = $TEST_DIRECTORY"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${TEST_DIRECTORY}" || exit 1
+find "${TEST_DIRECTORY}" -name "results.csv" \
+  -exec echo "-->" '{}' \; \
+  -exec column -t -s ',' '{}' \; \
+  -exec echo "========================================================" \; \
+  -exec echo "" \;
+
diff --git a/docker/scripts/split_kakfa_output_by_log.sh b/docker/scripts/split_kakfa_output_by_log.sh
new file mode 100755
index 0000000..71ec82b
--- /dev/null
+++ b/docker/scripts/split_kakfa_output_by_log.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2143,SC1083,SC2002,SC2126
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# For a given directory, finds all the bro log output, and splits the kafka output file
+# by bro log, such that there is a bro log -> bro log kafka log
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --log-directory                 [REQUIRED] The directory with the logs"
+  echo "    -h/--help                       Usage information."
+  echo " "
+  echo " "
+}
+
+LOG_DIRECTORY=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # LOG_DIRECTORY
+  #
+  #   --log-directory
+  #
+    --log-directory=*)
+      LOG_DIRECTORY="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$LOG_DIRECTORY" ]]; then
+  echo "$LOG_DIRECTORY must be passed"
+  exit 1
+fi
+
+echo "Running with "
+echo "$LOG_DIRECTORY = $LOG_DIRECTORY"
+echo "==================================================="
+
+# Move over to the docker area
+cd "${LOG_DIRECTORY}" || exit 1
+
+# for each log file, that is NOT KAFKA_OUTPUT_FILE we want to get the name
+# and extract the start
+# then we want to grep that name > name.kafka.log from the KAFKA_OUTPUT_FILE
+RESULTS_FILE="${LOG_DIRECTORY}/results.csv"
+echo "LOG,BRO_COUNT,KAFKA_COUNT" >> "${RESULTS_FILE}"
+for log in "${LOG_DIRECTORY}"/*.log
+do
+  BASE_LOG_FILE_NAME=$(basename "$log" .log)
+  if [[ ! "$BASE_LOG_FILE_NAME" == "kafka-output.log" ]]; then
+    if [[ $(grep {\""${BASE_LOG_FILE_NAME}"\": "${LOG_DIRECTORY}"/kafka-output.log) ]]; then
+      grep {\""${BASE_LOG_FILE_NAME}"\": "${LOG_DIRECTORY}"/kafka-output.log > "${LOG_DIRECTORY}"/"${BASE_LOG_FILE_NAME}".kafka.log
+
+      KAKFA_COUNT=$(cat "${LOG_DIRECTORY}/${BASE_LOG_FILE_NAME}.kafka.log" | wc -l)
+      BRO_COUNT=$(grep -v "#" "${log}" | wc -l)
+
+      echo "${BASE_LOG_FILE_NAME},${BRO_COUNT},${KAKFA_COUNT}" >> "${RESULTS_FILE}"
+    fi
+  fi
+done
+
diff --git a/docker/scripts/stop_container.sh b/docker/scripts/stop_container.sh
new file mode 100755
index 0000000..a26e2c7
--- /dev/null
+++ b/docker/scripts/stop_container.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+# Stops the Docker container with a given name
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --container-name                [REQUIRED] The Docker container name"
+  echo "    -h/--help                       Usage information."
+  echo " "
+}
+
+CONTAINER_NAME=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # CONTAINER_NAME
+  #
+  #   --container-name
+  #
+    --container-name=*)
+      CONTAINER_NAME="${i#*=}"
+      shift # past argument=value
+    ;;
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      echo "Error: unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+  echo "CONTAINER_NAME must be passed"
+  exit 1
+fi
+
+echo "Running stop_container with"
+echo "CONTAINER_NAME= $CONTAINER_NAME"
+echo "==================================================="
+
+docker stop "${CONTAINER_NAME}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
+
+docker rm "${CONTAINER_NAME}"
+
diff --git a/docker/test_output/.gitignore b/docker/test_output/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/docker/test_output/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file


Mime
View raw message