| #!/usr/bin/env bash |
| |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| shopt -s nocasematch |
| set -u # nounset |
| set -e # errexit |
| set -E # errtrap |
| set -o pipefail |
| |
| function help { |
| echo " " |
| echo "USAGE" |
| echo " --skip-docker-build [OPTIONAL] Skip build of bro docker machine." |
| echo " --data-path [OPTIONAL] The pcap data path. Default: ./data" |
| echo " --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro" |
| echo " -h/--help Usage information." |
| echo " " |
| echo "COMPATABILITY" |
| echo " bash >= 4.0 is required." |
| echo " " |
| } |
| |
| # Require bash >= 4 |
| if (( BASH_VERSINFO[0] < 4 )); then |
| >&2 echo "ERROR> bash >= 4.0 is required" >&2 |
| help |
| exit 1 |
| fi |
| |
| SKIP_REBUILD_BRO=false |
| |
| ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" |
| SCRIPT_DIR="${ROOT_DIR}"/scripts |
| CONTAINER_DIR="${ROOT_DIR}"/containers/bro-localbuild-container |
| DATA_PATH="${ROOT_DIR}"/data |
| DATE=$(date) |
| LOG_DATE=${DATE// /_} |
| TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_} |
| KAFKA_TOPIC="bro" |
| |
| # Handle command line options |
| for i in "$@"; do |
| case $i in |
| # |
| # SKIP_REBUILD_BRO |
| # |
| # --skip-docker-build |
| # |
| --skip-docker-build) |
| SKIP_REBUILD_BRO=true |
| shift # past argument |
| ;; |
| # |
| # DATA_PATH |
| # |
| --data-path=*) |
| DATA_PATH="${i#*=}" |
| shift # past argument=value |
| ;; |
| # |
| # KAFKA_TOPIC |
| # |
| # --kafka-topic |
| # |
| --kafka-topic=*) |
| KAFKA_TOPIC="${i#*=}" |
| shift # past argument=value |
| ;; |
| # |
| # -h/--help |
| # |
| -h | --help) |
| help |
| exit 0 |
| shift # past argument with no value |
| ;; |
| esac |
| done |
| |
| EXTRA_ARGS="$*" |
| |
| echo "Running build_container with " |
| echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO" |
| echo "DATA_PATH = $DATA_PATH" |
| echo "KAFKA_TOPIC = $KAFKA_TOPIC" |
| echo "===================================================" |
| |
| # Create the network |
| bash "${SCRIPT_DIR}"/create_docker_network.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Run the zookeeper container |
| bash "${SCRIPT_DIR}"/docker_run_zookeeper_container.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Wait for zookeeper to be up |
| bash "${SCRIPT_DIR}"/docker_run_wait_for_zookeeper.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Run the kafka container |
| bash "${SCRIPT_DIR}"/docker_run_kafka_container.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Wait for kafka to be up |
| bash "${SCRIPT_DIR}"/docker_run_wait_for_kafka.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Create the kafka topic |
| bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic=${KAFKA_TOPIC} |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Build the bro container |
| if [[ "$SKIP_REBUILD_BRO" = false ]]; then |
| bash "${SCRIPT_DIR}"/build_container.sh \ |
| --container-directory="${CONTAINER_DIR}" \ |
| --container-name=metron-bro-docker-container:latest |
| |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| fi |
| |
| # Download the pcaps |
| bash "${SCRIPT_DIR}"/download_sample_pcaps.sh --data-path="${DATA_PATH}" |
| |
| # By not catching $? here we are accepting that a failed pcap download will not |
| # exit the script |
| |
| mkdir "${TEST_OUTPUT_PATH}" || exit 1 |
| |
| # Run the bro container and optionally the passed script _IN_ the container |
| bash "${SCRIPT_DIR}"/docker_run_bro_container.sh \ |
| --data-path="${DATA_PATH}" \ |
| --test-output-path="${TEST_OUTPUT_PATH}" \ |
| "$EXTRA_ARGS" |
| |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| exit ${rc} |
| fi |
| |
| # Build the bro plugin |
| bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> FAILED TO BUILD PLUGIN. CHECK LOGS ${rc}" |
| exit ${rc} |
| fi |
| |
| # Configure it the bro plugin |
| bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> FAILED TO CONFIGURE PLUGIN. CHECK LOGS ${rc}" |
| exit ${rc} |
| fi |
| |
| |
| # for each pcap in the data directory, we want to |
| # run bro then read the output from kafka |
| # and output both of them to the same directory named |
| # for the date/pcap |
| |
| |
| for file in "${DATA_PATH}"/**/*.pcap* |
| do |
| # get the file name |
| BASE_FILE_NAME=$(basename "${file}") |
| DOCKER_DIRECTORY_NAME=${BASE_FILE_NAME//\./_} |
| |
| mkdir "${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" || exit 1 |
| echo "MADE ${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" |
| |
| # get the current offset in kafka |
| # this is where we are going to _start_ |
| OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic=${KAFKA_TOPIC} | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/") |
| echo "OFFSET------------------> ${OFFSET}" |
| |
| bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}" --output-directory-name="${DOCKER_DIRECTORY_NAME}" |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> FAILED TO PROCESS ${file} DATA. CHECK LOGS, please run the finish_end_to_end.sh when you are done." |
| exit ${rc} |
| fi |
| |
| KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log" |
| bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset=${OFFSET} --kafka-topic=${KAFKA_TOPIC} | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}" |
| |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS" |
| fi |
| |
| "${SCRIPT_DIR}"/split_kakfa_output_by_log.sh --log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> ISSUE ENCOUNTERED WHEN SPLITTING KAFKA OUTPUT LOGS" |
| fi |
| done |
| |
| "${SCRIPT_DIR}"/print_results.sh --test-directory="${TEST_OUTPUT_PATH}" |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> ISSUE ENCOUNTERED WHEN PRINTING RESULTS" |
| exit ${rc} |
| fi |
| |
| "${SCRIPT_DIR}"/analyze_results.sh --test-directory="${TEST_OUTPUT_PATH}" |
| rc=$?; if [[ ${rc} != 0 ]]; then |
| echo "ERROR> ISSUE ENCOUNTERED WHEN ANALYZING RESULTS" |
| exit ${rc} |
| fi |
| |
| echo "" |
| echo "Run complete" |
| echo "The kafka and bro output can be found at ${TEST_OUTPUT_PATH}" |
| echo "You may now work with the containers if you will. You need to call finish_end_to_end.sh when you are done" |