metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [metron-bro-plugin-kafka] branch master updated: METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29
Date Thu, 28 Feb 2019 18:56:08 GMT
This is an automated email from the ASF dual-hosted git repository.

otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
     new 2f1edcb  METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka
counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29
2f1edcb is described below

commit 2f1edcbec745d110ff9489dbf03348e428ea4c76
Author: JonZeolla <zeolla@gmail.com>
AuthorDate: Thu Feb 28 12:29:55 2019 -0500

    METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ
(JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29
---
 docker/README.md                            |  10 +-
 docker/run_end_to_end.sh                    |  16 ++-
 docker/scripts/analyze_results.sh           | 207 ++++++++++++++++++++++++++++
 docker/scripts/build_container.sh           |   4 +-
 docker/scripts/print_results.sh             |   5 +-
 docker/scripts/split_kakfa_output_by_log.sh |   3 +-
 6 files changed, 237 insertions(+), 8 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 8e4d3fa..3bae93b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -61,6 +61,7 @@ testing scripts to be added to a pull request, and subsequently to a test
suite.
 #### Scripts executed on the host to setup and interact with the docker containers
 
 ```bash
+├── analyze_results.sh
 ├── build_container.sh
 ├── cleanup_docker.sh
 ├── create_docker_network.sh
@@ -83,6 +84,11 @@ testing scripts to be added to a pull request, and subsequently to a test
suite.
 └── stop_container.sh
 ```
 
+- `analyze_results.sh`: Analyzes the `results.csv` files for any issues
+  ###### Parameters
+  ```bash
+  --test-directory               [REQUIRED] The directory for the tests
+  ```
 - `build_container.sh`: Runs docker build in the passed directory, and names the results
   ###### Parameters
   ```bash
@@ -191,12 +197,12 @@ testing scripts to be added to a pull request, and subsequently to a
test suite.
   ```bash
   --data-path                    [REQUIRED] The pcap data path
   ```
-- `print_results.sh` : Prints the `results.csv` for all the pcaps processed in the given
directory to console
+- `print_results.sh`: Prints the `results.csv` for all the pcaps processed in the given directory
to console
   ###### Parameters
   ```bash
   --test-directory               [REQUIRED] The directory for the tests
   ```
-- `split_kafka_output_by_log.sh` : For a pcap result directory, will create a LOG.kafka.log
for each LOG.log's entry in the kafka-output.log
+- `split_kafka_output_by_log.sh`: For a pcap result directory, will create a LOG.kafka.log
for each LOG.log's entry in the kafka-output.log
   ###### Parameters
   ```bash
   --log-directory                [REQUIRED] The directory with the logs
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 6baf679..ae06715 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -182,22 +182,34 @@ do
   echo "OFFSET------------------> ${OFFSET}"
 
   bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
-
   rc=$?; if [[ ${rc} != 0 ]]; then
     echo "ERROR> FAILED TO PROCESS ${file} DATA.  CHECK LOGS, please run the finish_end_to_end.sh
when you are done."
     exit ${rc}
   fi
+
   KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
   bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET | "${ROOT_DIR}"/remove_timeout_message.sh
| tee "${KAFKA_OUTPUT_FILE}"
-
   rc=$?; if [[ ${rc} != 0 ]]; then
     echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA.  CHECK LOGS"
   fi
 
   "${SCRIPT_DIR}"/split_kakfa_output_by_log.sh --log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+  rc=$?; if [[ ${rc} != 0 ]]; then
+    echo "ERROR> ISSUE ENCOUNTERED WHEN SPLITTING KAFKA OUTPUT LOGS"
+  fi
 done
 
 "${SCRIPT_DIR}"/print_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  echo "ERROR> ISSUE ENCOUNTERED WHEN PRINTING RESULTS"
+  exit ${rc}
+fi
+
+"${SCRIPT_DIR}"/analyze_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+  echo "ERROR> ISSUE ENCOUNTERED WHEN ANALYZING RESULTS"
+  exit ${rc}
+fi
 
 echo ""
 echo "Run complete"
diff --git a/docker/scripts/analyze_results.sh b/docker/scripts/analyze_results.sh
new file mode 100755
index 0000000..790ec18
--- /dev/null
+++ b/docker/scripts/analyze_results.sh
@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+shopt -s nocasematch
+#set -u # nounset disabled
+set -e # errexit
+set -E # errtrap
+set -o pipefail
+
+#
+# Analyzes the results.csv files to identify issues
+#
+
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "    --test-directory           [REQUIRED] The directory for the tests"
+  echo "    -h/--help                  Usage information."
+  echo " "
+  echo " "
+}
+
+function _echo() {
+  color="txt${1:-DEFAULT}"
+  case "${1}" in
+    ERROR)
+      >&2 echo -e "${!color}${1}> ${2}${txtDEFAULT}"
+      ;;
+    WARN)
+      echo -e "${!color}${1}> ${2}${txtDEFAULT}"
+      ;;
+    *)
+      echo -e "${!color}${1}> ${2}${txtDEFAULT}"
+      ;;
+  esac
+}
+
+SCRIPT_NAME=$(basename -- "$0")
+TEST_DIRECTORY=
+declare -A LOGS_WITH_UNEQUAL_RESULTS
+declare -a LOG_NAMES
+declare -A OVERALL_LOG_CARDINALITY
+declare -A LOG_ISSUE_COUNT
+declare -r txtDEFAULT='\033[0m'
+# shellcheck disable=SC2034
+declare -r txtERROR='\033[0;31m'
+# shellcheck disable=SC2034
+declare -r txtWARN='\033[0;33m'
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # TEST_DIRECTORY
+  #
+  #   --test-directory
+  #
+    --test-directory=*)
+      TEST_DIRECTORY="${i#*=}"
+      shift # past argument=value
+    ;;
+
+  #
+  # -h/--help
+  #
+    -h | --help)
+      help
+      exit 0
+      shift # past argument with no value
+    ;;
+
+  #
+  # Unknown option
+  #
+    *)
+      UNKNOWN_OPTION="${i#*=}"
+      _echo ERROR "unknown option: $UNKNOWN_OPTION"
+      help
+    ;;
+  esac
+done
+
+if [[ -z "$TEST_DIRECTORY" ]]; then
+  echo "$TEST_DIRECTORY must be passed"
+  exit 1
+fi
+
+echo "Running ${SCRIPT_NAME} with"
+echo "TEST_DIRECTORY = $TEST_DIRECTORY"
+echo "==================================================="
+
+## Main functions
+function count_occurrences_of_each_log_file
+{
+  # Count the number of occurences of each log name
+  for LOG_NAME in "${LOG_NAMES[@]}"; do
+    (( ++OVERALL_LOG_CARDINALITY["${LOG_NAME}"] ))
+  done
+}
+
+function check_for_unequal_log_counts
+{
+  RESULTS_FILE="${1}"
+
+  # Get the pcap folder name from the provided file
+  # shellcheck disable=SC2001
+  PCAP_FOLDER="$( cd "$( dirname "${RESULTS_FILE}" )" >/dev/null 2>&1 &&
echo "${PWD##*/}")"
+
+  # Check each log line in the provided log file for unequal results
+  for LOG_NAME in "${LOG_NAMES[@]}"; do
+    # For each log in the provided results, identify any unequal log counts
+    UNEQUAL_LOG=$(awk -F\, -v log_name="${LOG_NAME}" '$1 == log_name && $2 != $3
{print $1}' "${RESULTS_FILE}")
+
+    # Create a space separated list of unequal logs to simulate a
+    # multidimensional array
+    if [[ -n "${UNEQUAL_LOG}" ]]; then
+      if [[ "${#LOGS_WITH_UNEQUAL_RESULTS[${PCAP_FOLDER}]}" -eq 0 ]]; then
+        LOGS_WITH_UNEQUAL_RESULTS["${PCAP_FOLDER}"]="${UNEQUAL_LOG}"
+      else
+        LOGS_WITH_UNEQUAL_RESULTS["${PCAP_FOLDER}"]+=" ${UNEQUAL_LOG}"
+      fi
+    fi
+  done
+}
+
+function print_unequal_results
+{
+  # Output a table with the pcap file and log name details where the imbalance
+  # was detected
+  {
+  echo "PCAP FOLDER,LOG NAME"
+
+  for KEY in "${!LOGS_WITH_UNEQUAL_RESULTS[@]}"; do
+    # This must be done because we are simulating multidimensional arrays due to
+    # the lack of native bash support
+    for VALUE in ${LOGS_WITH_UNEQUAL_RESULTS[${KEY}]}; do
+      echo "${KEY},${VALUE}"
+    done
+  done
+  } | column -t -s ','
+}
+
+function print_log_comparison_insights
+{
+  # Load the log to instance count mapping from LOGS_WITH_UNEQUAL_RESULTS into a new
+  # associative array
+  # shellcheck disable=SC2046
+  declare -A $(echo "${LOGS_WITH_UNEQUAL_RESULTS[@]}" | tr ' ' '\n' | sort | uniq -c | awk
'{print "LOG_ISSUE_COUNT["$2"]="$1}')
+
+  # Compare each log type's instances of inequality to the total number of
+  # instances of each log.  If they are equal, this indicates that there may be
+  # a log-type related issue.
+  #
+  # For example, if count_occurrences_of_each_log_file identified that there
+  # were 10 instances of http logs across all of the `results.csv` files,
+  # ${OVERALL_LOG_CARDINALITY[http]} should equal 10. If check_for_unequal_log_counts
+  # independently found 10 instances where the http bro and kafka log counts
+  # from the `results.csv` files were not equal, ${LOG_ISSUE_COUNT[http]}
+  # would also have 10 entries, causing us to warn the user of that insight.
+  for KEY in "${!LOG_ISSUE_COUNT[@]}"; do
+    if [[ "${LOG_ISSUE_COUNT[${KEY}]}" == "${OVERALL_LOG_CARDINALITY[${KEY}]}" ]]; then
+      _echo WARN "None of the ${KEY} log counts were the same between bro and kafka.  This
may indicate an issue specific to that log."
+    fi
+  done
+}
+
+## Main
+# Move over to the docker area
+cd "${TEST_DIRECTORY}" || exit 1
+# Get a list of results files
+RESULTS_FILES=$(find "${TEST_DIRECTORY}" -name "results.csv")
+# Analyze each results file for issues
+for file in $RESULTS_FILES; do
+  # Capture the first column (the log names) of the provided file's contents in
+  # the array LOG_NAMES, excluding the header
+  mapfile -s 1 -t LOG_NAMES < <(awk -F\, '{print $1}' "${file}")
+
+  count_occurrences_of_each_log_file
+  check_for_unequal_log_counts "${file}"
+done
+
+if [[ "${#LOGS_WITH_UNEQUAL_RESULTS[@]}" -gt 0 ]]; then
+  _echo ERROR "UNEQUALITY FOUND IN BRO AND KAFKA LOG COUNTS"
+  echo ""
+
+  print_unequal_results
+  print_log_comparison_insights
+
+  exit 1
+fi
+
diff --git a/docker/scripts/build_container.sh b/docker/scripts/build_container.sh
index d4e5dca..40810db 100755
--- a/docker/scripts/build_container.sh
+++ b/docker/scripts/build_container.sh
@@ -22,6 +22,7 @@ set -u # nounset
 set -e # errexit
 set -E # errtrap
 set -o pipefail
+
 #
 # Runs docker build in a provided directory, with a provided name
 #
@@ -36,6 +37,7 @@ function help {
   echo " "
 }
 
+SCRIPT_NAME=$(basename -- "$0")
 CONTAINER_DIRECTORY=
 CONTAINER_NAME=
 
@@ -92,7 +94,7 @@ if [[ -z "$CONTAINER_NAME" ]]; then
   exit 1
 fi
 
-echo "Running with "
+echo "Running ${SCRIPT_NAME} with"
 echo "CONTAINER_DIRECTORY = $CONTAINER_DIRECTORY"
 echo "CONTAINER_NAME = $CONTAINER_NAME"
 echo "==================================================="
diff --git a/docker/scripts/print_results.sh b/docker/scripts/print_results.sh
index ecc67ca..6e107c7 100755
--- a/docker/scripts/print_results.sh
+++ b/docker/scripts/print_results.sh
@@ -31,11 +31,12 @@ function help {
   echo " "
   echo "usage: ${0}"
   echo "    --test-directory           [REQUIRED] The directory for the tests"
-  echo "    -h/--help                   Usage information."
+  echo "    -h/--help                  Usage information."
   echo " "
   echo " "
 }
 
+SCRIPT_NAME=$(basename -- "$0")
 TEST_DIRECTORY=
 
 # Handle command line options
@@ -77,7 +78,7 @@ if [[ -z "$TEST_DIRECTORY" ]]; then
 fi
 
 
-echo "Running with "
+echo "Running ${SCRIPT_NAME} with"
 echo "TEST_DIRECTORY = $TEST_DIRECTORY"
 echo "==================================================="
 
diff --git a/docker/scripts/split_kakfa_output_by_log.sh b/docker/scripts/split_kakfa_output_by_log.sh
index 74d55e3..61e53e4 100755
--- a/docker/scripts/split_kakfa_output_by_log.sh
+++ b/docker/scripts/split_kakfa_output_by_log.sh
@@ -37,6 +37,7 @@ function help {
   echo " "
 }
 
+SCRIPT_NAME=$(basename -- "$0")
 LOG_DIRECTORY=
 
 # Handle command line options
@@ -77,7 +78,7 @@ if [[ -z "$LOG_DIRECTORY" ]]; then
   exit 1
 fi
 
-echo "Running with "
+echo "Running ${SCRIPT_NAME} with"
 echo "$LOG_DIRECTORY = $LOG_DIRECTORY"
 echo "==================================================="
 


Mime
View raw message