metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [2/3] incubator-metron git commit: METRON-108 Create Fast Packet Capture Process (nickwallen) closes apache/incubator-metron#73
Date Wed, 13 Apr 2016 14:48:28 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dependencies.yml b/deployment/roles/packet-capture/tasks/dependencies.yml
new file mode 100644
index 0000000..4d6edc4
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dependencies.yml
@@ -0,0 +1,38 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+  - name: Install dependencies
+    yum: name={{ item }}
+    with_items:
+      - "@Development tools"
+      - pciutils
+      - net-tools
+      - glib2
+      - glib2-devel
+      - git
+
+  #
+  # install prerequisite packages and the latest kernel headers.  need to
+  # ensure that the kernel headers match the current running kernel version.
+  # if this is not the case, the DPDK build process will fail
+  #
+  - name: Install latest kernel headers and source
+    yum: name={{ item }} state=latest
+    with_items:
+      - kernel
+      - kernel-devel
+      - kernel-headers

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dpdk.yml b/deployment/roles/packet-capture/tasks/dpdk.yml
new file mode 100644
index 0000000..3780be7
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+  - name: "Download DPDK version {{ dpdk_version }}"
+    unarchive:
+      src: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
+      dest: "/root"
+      creates: "{{ dpdk_sdk }}"
+      copy: no
+
+  - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+    shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+    args:
+      chdir: "{{ dpdk_sdk }}"
+      creates: "{{ dpdk_home }}"
+
+  - name: "Turn on debug flags"
+    lineinfile:
+      dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+      regexp: 'DEBUG=n'
+      line: 'DEBUG=y'
+    tags:
+      - debug
+
+  - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+    shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+    args:
+      chdir: "{{ dpdk_sdk }}"
+      creates: "{{ dpdk_home }}"
+
+  - name: Load kernel modules to enable userspace IO
+    shell: "{{ item }}"
+    with_items:
+      - modprobe uio_pci_generic
+      - modprobe vfio-pci
+
+  - name: Bind the device to the loaded kernel module(s)
+    shell: "{{ dpdk_home }}/sbin/dpdk_nic_bind --force --bind=uio_pci_generic {{ item }}"
+    with_items: "{{ dpdk_device }}"
+
+  - name: Set useful environment variables
+    lineinfile: "dest=/root/.bash_profile line={{ item }}"
+    with_items:
+      - "export RTE_SDK={{ dpdk_sdk }}"
+      - "export RTE_TARGET={{ dpdk_target }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/kernel.yml b/deployment/roles/packet-capture/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters.  set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+  - set_fact:
+      expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+  - name: Check kernel boot parameters
+    shell: "cat /proc/cmdline"
+    register: actual_kernel_params
+
+  - name: Alter kernel boot parameters
+    lineinfile:
+      dest: /etc/default/grub
+      regexp:  '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+      line: '\1 {{ expected_kernel_params }}"'
+      backrefs: yes
+    when: not expected_kernel_params in actual_kernel_params.stdout
+
+  - name: Update grub with kernel boot parameters
+    shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+    when: not expected_kernel_params in actual_kernel_params.stdout
+
+  - name: Restart for modified kernel params
+    command: shutdown -r now "modified kernel params"
+    async: 0
+    poll: 0
+    ignore_errors: true
+    when: not expected_kernel_params in actual_kernel_params.stdout
+    
+  - name: Wait for reboot of '{{ inventory_hostname }}'
+    local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+    become: false
+    when: not expected_kernel_params in actual_kernel_params.stdout

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/main.yml b/deployment/roles/packet-capture/tasks/main.yml
new file mode 100644
index 0000000..f096178
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+  - include: dependencies.yml
+  - include: kernel.yml
+  - include: dpdk.yml
+  - include: pcapture.yml
+  - include: debug.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/pcapture.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/pcapture.yml b/deployment/roles/packet-capture/tasks/pcapture.yml
new file mode 100644
index 0000000..d00d379
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/pcapture.yml
@@ -0,0 +1,49 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Distribute pcapture
+  copy: src=../../../metron-sensors/packet-capture dest={{ pcapture_work_dir | dirname }} mode=0755
+
+- name: Build pcapture
+  shell: "{{ item }}"
+  args:
+    chdir: "{{ pcapture_work_dir }}"
+  with_items:
+    - make
+  environment:
+    RTE_SDK: "{{ dpdk_sdk }}"
+    RTE_TARGET: "{{ dpdk_target }}"
+    LD_LIBRARY_PATH: "{{ pcapture_ld_library_path }}"
+
+- name: Install pcapture
+  shell: "cp {{ pcapture_work_dir }}/src/build/app/{{ pcapture_bin }} {{ pcapture_prefix }}"
+  args:
+    chdir: "{{ pcapture_work_dir }}"
+    creates: "{{ pcapture_prefix }}/{{ pcapture_bin }}"
+
+- name: Deploy configuration
+  template: src=pcapture.conf dest={{ pcapture_kafka_config }} mode=0755
+
+- name: Deploy service
+  template: src=pcapture dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+  shell: systemctl enable pcapture
+  when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
+
+- name: Run pcapture
+  service: name=pcapture state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture b/deployment/roles/packet-capture/templates/pcapture
new file mode 100644
index 0000000..8c2221a
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcapture daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: pcapture
+#
+
+export RTE_SDK="{{ dpdk_sdk }}"
+export RTE_TARGET="{{ dpdk_target }}"
+export LD_LIBRARY_PATH="{{ pcapture_ld_library_path }}"
+
+DAEMON_PATH="{{ dpdk_sdk }}"
+DAEMON="{{ pcapture_prefix }}/{{ pcapture_bin }}"
+DAEMONOPTS+=" -- "
+DAEMONOPTS+="-p {{ pcapture_portmask }} "
+DAEMONOPTS+="-t {{ pcapture_topic }} "
+DAEMONOPTS+="-c {{ pcapture_kafka_config }} "
+
+NAME="pcapture"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMONLOG=/var/log/$NAME.log
+NOW=`date`
+
+case "$1" in
+  start)
+    printf "%-50s" "Starting $NAME..."
+    echo "$NOW:  Starting $NAME..." >> $DAEMONLOG
+    cd $DAEMON_PATH
+    PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
+    if [ -z $PID ]; then
+        printf "%s\n" "Fail"
+    else
+        echo $PID > $PIDFILE
+        printf "%s\n" "Ok"
+    fi
+  ;;
+
+  status)
+    printf "%-50s" "Checking $NAME..."
+    if [ -f $PIDFILE ]; then
+      PID=`cat $PIDFILE`
+      if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+        printf "%s\n" "Process dead but pidfile exists"
+      else
+        echo "Running"
+      fi
+    else
+      printf "%s\n" "Service not running"
+    fi
+  ;;
+
+  stop)
+    printf "%-50s" "Stopping $NAME"
+    PID=`cat $PIDFILE`
+    cd $DAEMON_PATH
+    if [ -f $PIDFILE ]; then
+        echo "$NOW:  Stopping $NAME with pid=$PID" >> $DAEMONLOG
+        kill -HUP $PID
+        printf "%s\n" "Ok"
+        rm -f $PIDFILE
+    else
+        printf "%s\n" "pidfile not found"
+    fi
+  ;;
+
+  restart)
+    $0 stop
+    $0 start
+  ;;
+
+  *)
+    echo "Usage: $0 {status|start|stop|restart}"
+    exit 1
+esac

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture.conf b/deployment/roles/packet-capture/templates/pcapture.conf
new file mode 100644
index 0000000..e404476
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture.conf
@@ -0,0 +1,67 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-packet-capture
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/Vagrantfile
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/Vagrantfile b/deployment/vagrant/packet-capture/Vagrantfile
new file mode 100644
index 0000000..1303712
--- /dev/null
+++ b/deployment/vagrant/packet-capture/Vagrantfile
@@ -0,0 +1,69 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+  # enable hostmanager
+  config.hostmanager.enabled = true
+  config.hostmanager.manage_host = true
+
+  #
+  # source
+  #
+  config.vm.define "source" do |node|
+
+    # host settings
+    node.vm.hostname = "source"
+    node.vm.box = "bento/centos-7.1"
+    node.ssh.insert_key = "true"
+    node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+    # provider
+    node.vm.provider "virtualbox" do |vb|
+      vb.memory = 1024
+      vb.cpus = 1
+    end
+  end
+
+  #
+  # sink
+  #
+  config.vm.define "sink" do |node|
+
+    # host settings
+    node.vm.hostname = "sink"
+    node.vm.box = "bento/centos-7.1"
+    node.ssh.insert_key = "true"
+    node.vm.network "public_network"
+    node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+    # provider
+    node.vm.provider "virtualbox" do |vb|
+      vb.memory = 4096
+      vb.cpus = 3
+
+      # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+      vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+      vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+    end
+  end
+
+  # provision hosts
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "playbook.yml"
+  end
+end

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/ansible.cfg b/deployment/vagrant/packet-capture/ansible.cfg
new file mode 100644
index 0000000..9c650c2
--- /dev/null
+++ b/deployment/vagrant/packet-capture/ansible.cfg
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles
+pipelining = True

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/playbook.yml b/deployment/vagrant/packet-capture/playbook.yml
new file mode 100644
index 0000000..7a5128c
--- /dev/null
+++ b/deployment/vagrant/packet-capture/playbook.yml
@@ -0,0 +1,43 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+#
+# produces network traffic
+#
+- hosts: source
+  become: yes
+  vars:
+    pcap_replay_interface: "enp0s8"
+  roles:
+    - role: pcap_replay
+
+#
+# consumes network traffic
+#
+- hosts: sink
+  become: yes
+  vars:
+      dpdk_device: ["00:08.0"]
+      dpdk_target: "x86_64-native-linuxapp-gcc"
+      num_huge_pages: 512
+      pcapture_portmask: 0xf
+      pcapture_topic: pcap
+      kafka_broker_url: localhost:9092
+  roles:
+    - role: librdkafka
+    - role: kafka-broker
+    - role: packet-capture

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/.gitignore b/metron-sensors/bro-plugin-kafka/.gitignore
new file mode 100644
index 0000000..28a8358
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/.gitignore
@@ -0,0 +1,31 @@
+.state
+build
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CHANGES
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CHANGES b/metron-sensors/bro-plugin-kafka/CHANGES
new file mode 100644
index 0000000..d9e26de
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CHANGES
@@ -0,0 +1,16 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CMakeLists.txt b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
new file mode 100644
index 0000000..30bf3b5
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
@@ -0,0 +1,43 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+cmake_minimum_required(VERSION 2.8)
+project(Plugin)
+include(BroPlugin)
+find_package(LibRDKafka)
+find_package(OpenSSL)
+
+if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
+  include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
+  bro_plugin_begin(BRO KAFKA)
+  bro_plugin_cc(src/KafkaWriter.cc)
+  bro_plugin_cc(src/Plugin.cc)
+  bro_plugin_cc(src/TaggedJSON.cc)
+  bro_plugin_bif(src/kafka.bif)
+  bro_plugin_dist_files(README CHANGES COPYING VERSION)
+  bro_plugin_link_library(${LibRDKafka_LIBRARIES})
+  bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
+  bro_plugin_link_library(${OpenSSL_LIBRARIES})
+  bro_plugin_end()
+
+elseif (NOT LIBRDKAFKA_FOUND)
+  message(FATAL_ERROR "LibRDKafka not found.")
+
+elseif (NOT OPENSSL_FOUND)
+  message(FATAL_ERROR "OpenSSL not found.")
+
+endif ()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/COPYING
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/COPYING b/metron-sensors/bro-plugin-kafka/COPYING
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/COPYING
@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/MAINTAINER
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/MAINTAINER b/metron-sensors/bro-plugin-kafka/MAINTAINER
new file mode 100644
index 0000000..0ddaed7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/MAINTAINER
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+Apache Metron <user@metron.incubator.apache.org>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/Makefile b/metron-sensors/bro-plugin-kafka/Makefile
new file mode 100644
index 0000000..50fa3ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/Makefile
@@ -0,0 +1,44 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# Convenience Makefile providing a few common top-level targets.
+#
+
+cmake_build_dir=build
+arch=`uname -s | tr A-Z a-z`-`uname -m`
+
+all: build-it
+
+build-it:
+	@test -e $(cmake_build_dir)/config.status || ./configure
+	-@test -e $(cmake_build_dir)/CMakeCache.txt && \
+      test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \
+      echo Updating stale CMake cache && \
+      touch $(cmake_build_dir)/CMakeCache.txt
+
+	( cd $(cmake_build_dir) && make )
+
+install:
+	( cd $(cmake_build_dir) && make install )
+
+clean:
+	( cd $(cmake_build_dir) && make clean )
+
+distclean:
+	rm -rf $(cmake_build_dir)
+
+test:
+	make -C tests

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/README
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/README b/metron-sensors/bro-plugin-kafka/README
new file mode 100644
index 0000000..e6caa7a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/README
@@ -0,0 +1,92 @@
+Bro Logging Output to Kafka
+===========================
+
+A Bro log writer that sends logging output to Kafka.  This provides a convenient
+means for tools in the Hadoop ecosystem, such as Storm, Spark, and others, to
+process the data generated by Bro.
+
+Installation
+------------
+
+Install librdkafka (https://github.com/edenhill/librdkafka), a native client
+library for Kafka.  This plugin has been tested against the latest release of
+librdkafka, which at the time of this writing is v0.8.6.
+
+    # curl -L https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz | tar xvz
+    # cd librdkafka-0.8.6/
+    # ./configure
+    # make
+    # sudo make install
+
+Then compile this Bro plugin using the following commands.
+
+    # ./configure --bro-dist=$BRO_SRC
+    # make
+    # sudo make install
+
+Run the following command to ensure that the plugin was installed successfully.
+
+    # bro -N Bro::Kafka
+    Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+
+Activation
+----------
+
+The easiest way to enable Kafka output is to load the plugin's
+``logs-to-kafka.bro`` script.  If you are using BroControl, the following lines
+added to local.bro will activate it.
+
+```
+@load Bro/Kafka/logs-to-kafka.bro
+redef Kafka::logs_to_send = set(Conn::LOG, HTTP::LOG, DNS::LOG);
+redef Kafka::topic_name = "bro";
+redef Kafka::kafka_conf = table(
+    ["metadata.broker.list"] = "localhost:9092"
+);
+```
+
+This example will send all HTTP, DNS, and Conn logs to a Kafka broker running on
+the localhost to a topic called ``bro``. Any configuration value accepted by
+librdkafka can be added to the ``kafka_conf`` configuration table.
+
+Settings
+--------
+
+### ``kafka_conf``
+
+The global configuration settings for Kafka.  These values are passed through
+directly to librdkafka.  Any valid librdkafka settings can be defined in this
+table.
+
+```
+redef Kafka::kafka_conf = table(
+    ["metadata.broker.list"] = "localhost:9092",
+    ["client.id"] = "bro"
+);
+```
+
+### ``topic_name``
+
+The name of the topic in Kafka where all Bro logs will be sent to.
+
+```
+redef Kafka::topic_name = "bro";
+```
+
+### ``max_wait_on_shutdown``
+
+The maximum number of milliseconds that the plugin will wait for any backlog of
+queued messages to be sent to Kafka before forced shutdown.
+
+```
+redef Kafka::max_wait_on_shutdown = 3000;
+```
+
+### ``tag_json``
+
+If true, a log stream identifier is appended to each JSON-formatted message. For
+example, a Conn::LOG message will look like ``{ 'conn' : { ... }}``.
+
+```
+redef Kafka::tag_json = T;
+```

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/VERSION
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/VERSION b/metron-sensors/bro-plugin-kafka/VERSION
new file mode 100644
index 0000000..204e5ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/VERSION
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+0.1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
new file mode 100644
index 0000000..c64d8f9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
@@ -0,0 +1,49 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+find_path(LibRDKafka_ROOT_DIR
+  NAMES include/librdkafka/rdkafkacpp.h
+)
+
+find_library(LibRDKafka_LIBRARIES
+  NAMES rdkafka++
+  HINTS ${LibRDKafka_ROOT_DIR}/lib
+)
+
+find_library(LibRDKafka_C_LIBRARIES
+	NAMES rdkafka
+	HINTS ${LibRDKafka_ROT_DIR}/lib
+)
+
+find_path(LibRDKafka_INCLUDE_DIR
+  NAMES librdkafka/rdkafkacpp.h
+  HINTS ${LibRDKafka_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibRDKafka DEFAULT_MSG
+  LibRDKafka_LIBRARIES
+  LibRDKafka_C_LIBRARIES
+  LibRDKafka_INCLUDE_DIR
+)
+
+mark_as_advanced(
+  LibRDKafka_ROOT_DIR
+  LibRDKafka_LIBRARIES
+  LibRDKafka_C_LIBRARIES
+  LibRDKafka_INCLUDE_DIR
+)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
new file mode 100644
index 0000000..5ed955c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
@@ -0,0 +1,72 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# - Try to find openssl include dirs and libraries
+#
+# Usage of this module as follows:
+#
+#     find_package(OpenSSL)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  OpenSSL_ROOT_DIR          Set this variable to the root installation of
+#                            openssl if the module has problems finding the
+#                            proper installation path.
+#
+# Variables defined by this module:
+#
+#  OPENSSL_FOUND             System has openssl, include and library dirs found
+#  OpenSSL_INCLUDE_DIR       The openssl include directories.
+#  OpenSSL_LIBRARIES         The openssl libraries.
+#  OpenSSL_CYRPTO_LIBRARY    The openssl crypto library.
+#  OpenSSL_SSL_LIBRARY       The openssl ssl library.
+
+find_path(OpenSSL_ROOT_DIR
+    NAMES include/openssl/ssl.h
+)
+
+find_path(OpenSSL_INCLUDE_DIR
+    NAMES openssl/ssl.h
+    HINTS ${OpenSSL_ROOT_DIR}/include
+)
+
+find_library(OpenSSL_SSL_LIBRARY
+    NAMES ssl ssleay32 ssleay32MD
+    HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+find_library(OpenSSL_CRYPTO_LIBRARY
+    NAMES crypto
+    HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+set(OpenSSL_LIBRARIES ${OpenSSL_SSL_LIBRARY} ${OpenSSL_CRYPTO_LIBRARY}
+    CACHE STRING "OpenSSL SSL and crypto libraries" FORCE)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OpenSSL DEFAULT_MSG
+    OpenSSL_LIBRARIES
+    OpenSSL_INCLUDE_DIR
+)
+
+mark_as_advanced(
+    OpenSSL_ROOT_DIR
+    OpenSSL_INCLUDE_DIR
+    OpenSSL_LIBRARIES
+    OpenSSL_CRYPTO_LIBRARY
+    OpenSSL_SSL_LIBRARY
+)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure b/metron-sensors/bro-plugin-kafka/configure
new file mode 100755
index 0000000..d053488
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# Wrapper for viewing/setting options that the plugin's CMake
+# scripts will recognize.
+#
+# Don't edit this. Edit configure.plugin to add plugin-specific options.
+#
+
+set -e
+command="$0 $*"
+
+if [ -e `dirname $0`/configure.plugin ]; then
+    # Include custom additions.
+    . `dirname $0`/configure.plugin
+fi
+
+# Check for `cmake` command.
+type cmake > /dev/null 2>&1 || {
+    echo "\
+This package requires CMake, please install it first, then you may
+use this configure script to access CMake equivalent functionality.\
+" >&2;
+    exit 1;
+}
+
+usage() {
+
+cat 1>&2 <<EOF
+Usage: $0 [OPTIONS]
+
+  Plugin Options:
+    --bro-dist=DIR             Path to Bro source tree
+    --install-root=DIR         Path where to install plugin into
+EOF
+
+if type plugin_usage >/dev/null 2>&1; then
+    plugin_usage 1>&2
+fi
+
+echo
+
+exit 1
+}
+
+# Function to append a CMake cache entry definition to the
+# CMakeCacheEntries variable
+#   $1 is the cache entry variable name
+#   $2 is the cache entry variable type
+#   $3 is the cache entry variable value
+append_cache_entry () {
+    CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
+}
+
+# set defaults
+builddir=build
+brodist=`cd ../../.. && pwd`
+installroot="default"
+CMakeCacheEntries=""
+
+while [ $# -ne 0 ]; do
+    case "$1" in
+        -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+        *) optarg= ;;
+    esac
+
+    case "$1" in
+        --help|-h)
+            usage
+            ;;
+        --bro-dist=*)
+            brodist=`cd $optarg && pwd`
+            ;;
+        --install-root=*)
+            installroot=$optarg
+            ;;
+        --with-openssl=*)
+            append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+            ;;
+        *)
+            if type plugin_option >/dev/null 2>&1; then
+                plugin_option $1 && shift && continue;
+            fi
+
+            echo "Invalid option '$1'.  Try $0 --help to see available options."
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+if [ ! -e "$brodist/bro-path-dev.in" ]; then
+    echo "Cannot determine Bro source directory, use --bro-dist=DIR."
+    exit 1
+fi
+
+append_cache_entry BRO_DIST PATH $brodist
+append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
+
+if [ "$installroot" != "default" ]; then
+    mkdir -p $installroot
+    append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot
+fi
+
+echo "Build Directory        : $builddir"
+echo "Bro Source Directory   : $brodist"
+
+mkdir -p $builddir
+cd $builddir
+
+cmake $CMakeCacheEntries ..
+
+echo "# This is the command used to configure this build" > config.status
+echo $command >> config.status
+chmod u+x config.status

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure.plugin
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure.plugin b/metron-sensors/bro-plugin-kafka/configure.plugin
new file mode 100644
index 0000000..1cb2086
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure.plugin
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# Hooks to add custom options to the configure script.
+#
+
+plugin_usage()
+{
+  cat <<EOF
+  --with-librdkafka=PATH	 path to librdkafka
+  --with-openssl=PATH      path to OpenSSL install root
+EOF
+}
+
+plugin_option()
+{
+  case "$1" in
+    --with-librdkafka=*)
+      append_cache_entry LibRdKafka_ROOT_DIR PATH $optarg
+      ;;
+    --with-openssl=*)
+      append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+      ;;
+    *)
+      return 1;
+    ;;
+    esac
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
new file mode 100644
index 0000000..12295a9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# This is loaded when a user activates the plugin. Include scripts here that should be
+# loaded automatically at that point.
+#

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
new file mode 100644
index 0000000..84e390c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
@@ -0,0 +1,44 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+##! load this script to enable log output to kafka
+
+module Kafka;
+
+export {
+	##
+	## which log streams should be sent to kafka?
+	## example:
+	##		redef Kafka::logs_to_send = set(Conn::Log, HTTP::LOG, DNS::LOG);
+	##
+	const logs_to_send: set[Log::ID] &redef;
+}
+
+event bro_init() &priority=-5
+{
+	for (stream_id in Log::active_streams)
+	{
+		if (stream_id in Kafka::logs_to_send)
+		{
+			local filter: Log::Filter = [
+				$name = fmt("kafka-%s", stream_id),
+				$writer = Log::WRITER_KAFKAWRITER
+			];
+
+			Log::add_filter(stream_id, filter);
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
new file mode 100644
index 0000000..fee9549
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# This is loaded unconditionally at Bro startup. Include scripts here that should
+# always be loaded.
+#
+# Normally, that will be only code that initializes built-in elements. Load
+# your standard scripts in
+# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
+#
+
+@load ./init.bro

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/init.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/init.bro b/metron-sensors/bro-plugin-kafka/scripts/init.bro
new file mode 100644
index 0000000..c76b2a6
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/init.bro
@@ -0,0 +1,27 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+module Kafka;
+
+export {
+  const topic_name: string = "bro" &redef;
+  const max_wait_on_shutdown: count = 3000 &redef;
+  const tag_json: bool = F &redef;
+  const kafka_conf: table[string] of string = table(
+    ["metadata.broker.list"] = "localhost:9092"
+  ) &redef;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
new file mode 100644
index 0000000..9019790
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Type.h>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+#include "kafka.bif.h"
+#include "TaggedJSON.h"
+#include "KafkaWriter.h"
+
+using namespace logging;
+using namespace writer;
+
+KafkaWriter::KafkaWriter(WriterFrontend* frontend): WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL)
+{
+    // TODO do we need this??
+    topic_name.assign((const char*)BifConst::Kafka::topic_name->Bytes(),
+        BifConst::Kafka::topic_name->Len());
+}
+
+KafkaWriter::~KafkaWriter()
+{}
+
+bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
+{
+    // initialize the formatter
+    if(BifConst::Kafka::tag_json) {
+      formatter = new threading::formatter::TaggedJSON(info.path, this, threading::formatter::JSON::TS_EPOCH);
+    } else {
+      formatter = new threading::formatter::JSON(this, threading::formatter::JSON::TS_EPOCH);
+    }
+
+    // kafka global configuration
+    string err;
+    conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+    // apply the user-defined settings to kafka
+    Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
+    IterCookie* c = val->AsTable()->InitForIteration();
+    HashKey* k;
+    TableEntryVal* v;
+    while ((v = val->AsTable()->NextEntry(k, c))) {
+
+        // fetch the key and value
+        ListVal* index = val->AsTableVal()->RecoverIndex(k);
+        string key = index->Index(0)->AsString()->CheckString();
+        string val = v->Value()->AsString()->CheckString();
+
+        // apply setting to kafka
+        if (RdKafka::Conf::CONF_OK != conf->set(key, val, err)) {
+            reporter->Error("Failed to set '%s'='%s': %s", key.c_str(), val.c_str(), err.c_str());
+            return false;
+        }
+
+        // cleanup
+        Unref(index);
+        delete k;
+    }
+
+    // create kafka producer
+    producer = RdKafka::Producer::create(conf, err);
+    if (!producer) {
+        reporter->Error("Failed to create producer: %s", err.c_str());
+        return false;
+    }
+
+    // create handle to topic
+    topic_conf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
+    topic = RdKafka::Topic::create(producer, topic_name, topic_conf, err);
+    if (!topic) {
+        reporter->Error("Failed to create topic handle: %s", err.c_str());
+        return false;
+    }
+
+    return true;
+}
+
+/**
+ * Writer-specific method called just before the threading system is
+ * going to shutdown. It is assumed that once this messages returns,
+ * the thread can be safely terminated.
+ */
+bool KafkaWriter::DoFinish(double network_time)
+{
+    bool success = false;
+    int poll_interval = 1000;
+    int waited = 0;
+    int max_wait = BifConst::Kafka::max_wait_on_shutdown;
+
+    // wait a bit for queued messages to be delivered
+    while (producer->outq_len() > 0 && waited <= max_wait) {
+        producer->poll(poll_interval);
+        waited += poll_interval;
+    }
+
+    // successful only if all messages delivered
+    if (producer->outq_len() == 0) {
+        reporter->Error("Unable to deliver %0d message(s)", producer->outq_len());
+        success = true;
+    }
+
+    delete topic;
+    delete producer;
+    delete formatter;
+
+    return success;
+}
+
+/**
+ * Writer-specific output method implementing recording of one log
+ * entry.
+ */
+bool KafkaWriter::DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals)
+{
+    ODesc buff;
+    buff.Clear();
+
+    // format the log entry
+    formatter->Describe(&buff, num_fields, fields, vals);
+
+    // send the formatted log entry to kafka
+    const char* raw = (const char*)buff.Bytes();
+    RdKafka::ErrorCode resp = producer->produce(
+        topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY,
+        const_cast<char*>(raw), strlen(raw), NULL, NULL);
+
+    if (RdKafka::ERR_NO_ERROR == resp) {
+        producer->poll(0);
+    }
+    else {
+        string err = RdKafka::err2str(resp);
+        reporter->Error("Kafka send failed: %s", err.c_str());
+    }
+
+    return true;
+}
+
+/**
+ * Writer-specific method implementing a change of fthe buffering
+ * state.	If buffering is disabled, the writer should attempt to
+ * write out information as quickly as possible even if doing so may
+ * have a performance impact. If enabled (which is the default), it
+ * may buffer data as helpful and write it out later in a way
+ * optimized for performance. The current buffering state can be
+ * queried via IsBuf().
+ */
+bool KafkaWriter::DoSetBuf(bool enabled)
+{
+    // no change in behavior
+    return true;
+}
+
+/**
+ * Writer-specific method implementing flushing of its output.	A writer
+ * implementation must override this method but it can just
+ * ignore calls if flushing doesn't align with its semantics.
+ */
+bool KafkaWriter::DoFlush(double network_time)
+{
+    producer->poll(0);
+    return true;
+}
+
+/**
+ * Writer-specific method implementing log rotation.	Most directly
+ * this only applies to writers writing into files, which should then
+ * close the current file and open a new one.	However, a writer may
+ * also trigger other apppropiate actions if semantics are similar.
+ * Once rotation has finished, the implementation *must* call
+ * FinishedRotation() to signal the log manager that potential
+ * postprocessors can now run.
+ */
+bool KafkaWriter::DoRotate(const char* rotated_path, double open, double close, bool terminating)
+{
+    // no need to perform log rotation
+    return FinishedRotation();
+}
+
+/**
+ * Triggered by regular heartbeat messages from the main thread.
+ */
+bool KafkaWriter::DoHeartbeat(double network_time, double current_time)
+{
+    producer->poll(0);
+    return true;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
new file mode 100644
index 0000000..2299667
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+
+#include <string>
+#include <librdkafka/rdkafkacpp.h>
+#include <logging/WriterBackend.h>
+#include <threading/formatters/JSON.h>
+#include <Type.h>
+#include "kafka.bif.h"
+
+#include "TaggedJSON.h"
+
+namespace logging { namespace writer {
+
+/**
+ * A logging writer that sends data to a Kafka broker.
+ */
+class KafkaWriter : public WriterBackend {
+
+public:
+    KafkaWriter(WriterFrontend* frontend);
+    ~KafkaWriter();
+
+    static WriterBackend* Instantiate(WriterFrontend* frontend)
+    {
+        return new KafkaWriter(frontend);
+    }
+
+protected:
+    virtual bool DoInit(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
+    virtual bool DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals);
+    virtual bool DoSetBuf(bool enabled);
+    virtual bool DoRotate(const char* rotated_path, double open, double close, bool terminating);
+    virtual bool DoFlush(double network_time);
+    virtual bool DoFinish(double network_time);
+    virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+    string topic_name;
+    threading::formatter::Formatter *formatter;
+    RdKafka::Producer* producer;
+    RdKafka::Topic* topic;
+    RdKafka::Conf* conf;
+    RdKafka::Conf* topic_conf;
+};
+
+}}
+
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.cc b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
new file mode 100644
index 0000000..d523d23
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Plugin.h"
+#include "KafkaWriter.h"
+
+namespace plugin { namespace Bro_Kafka {
+    Plugin plugin;
+}}
+
+using namespace plugin::Bro_Kafka;
+
+plugin::Configuration Plugin::Configure()
+{
+    AddComponent(new ::logging::Component("KafkaWriter", ::logging::writer::KafkaWriter::Instantiate));
+
+    plugin::Configuration config;
+    config.name = "Bro::Kafka";
+    config.description = "Writes logs to Kafka";
+    config.version.major = 0;
+    config.version.minor = 1;
+    return config;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.h b/metron-sensors/bro-plugin-kafka/src/Plugin.h
new file mode 100644
index 0000000..8adeb18
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.h
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA
+#define BRO_PLUGIN_BRO_KAFKA
+
+#include <plugin/Plugin.h>
+
+namespace plugin { namespace Bro_Kafka {
+
+    class Plugin : public ::plugin::Plugin {
+    protected:
+        // Overridden from plugin::Plugin.
+        virtual plugin::Configuration Configure();
+    };
+
+    extern Plugin plugin;
+}}
+
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
new file mode 100644
index 0000000..db3f305
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TaggedJSON.h"
+
+namespace threading { namespace formatter {
+
+TaggedJSON::TaggedJSON(string sn, MsgThread* t, JSON::TimeFormat tf): JSON(t, tf), stream_name(sn)
+{}
+
+TaggedJSON::~TaggedJSON()
+{}
+
+bool TaggedJSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const
+{
+    desc->AddRaw("{");
+
+    // 'tag' the json; aka prepend the stream name to the json-formatted log content
+    desc->AddRaw("\"");
+    desc->AddRaw(stream_name);
+    desc->AddRaw("\": ");
+
+    // append the JSON formatted log record itself
+    JSON::Describe(desc, num_fields, fields, vals);
+
+    desc->AddRaw("}");
+    return true;
+}
+}}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
new file mode 100644
index 0000000..08a50df
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+
+#include <string>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+
+using threading::formatter::JSON;
+using threading::MsgThread;
+using threading::Value;
+using threading::Field;
+
+namespace threading { namespace formatter {
+
+/*
+ * A JSON formatter that prepends or 'tags' the content with a log stream
+ * identifier.  For example,
+ *   { 'conn' : { ... }}
+ *   { 'http' : { ... }}
+ */
+class TaggedJSON : public JSON {
+
+public:
+    TaggedJSON(string stream_name, MsgThread* t, JSON::TimeFormat tf);
+    virtual ~TaggedJSON();
+    virtual bool Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const;
+
+private:
+    string stream_name;
+};
+
+}}
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka.bif b/metron-sensors/bro-plugin-kafka/src/kafka.bif
new file mode 100644
index 0000000..8a8070c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka.bif
@@ -0,0 +1,23 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+module Kafka;
+
+const kafka_conf: config;
+const topic_name: string;
+const max_wait_on_shutdown: count;
+const tag_json: bool;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka_const.bif b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
new file mode 100644
index 0000000..989c0ae
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+module Kafka;
+
+type config : table[string] of string;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Makefile b/metron-sensors/bro-plugin-kafka/tests/Makefile
new file mode 100644
index 0000000..a637cd3
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Makefile
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+test:
+	@btest

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
new file mode 100755
index 0000000..8aa0ea7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
@@ -0,0 +1,36 @@
+#! /bin/sh
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# BTest helper for getting values for Bro-related environment variables.
+
+base=`dirname $0`
+bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
+
+if [ "$1" = "brobase" ]; then
+    echo ${bro}
+elif [ "$1" = "bropath" ]; then
+    ${bro}/build/bro-path-dev
+elif [ "$1" = "bro_plugin_path" ]; then
+    ( cd ${base}/../.. && pwd )
+elif [ "$1" = "bro_seed_file" ]; then
+    echo ${bro}/testing/btest/random.seed
+elif [ "$1" = "path" ]; then
+    echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
+else
+    echo "usage: `basename $0` <var>" >&2
+    exit 1
+fi

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/btest.cfg
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/btest.cfg b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
new file mode 100644
index 0000000..e42fefd
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
@@ -0,0 +1,36 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+[btest]
+TestDirs    = kafka
+TmpDir      = %(testbase)s/.tmp
+BaselineDir = %(testbase)s/Baseline
+IgnoreDirs  = .svn CVS .tmp
+IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
+
+[environment]
+BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
+BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
+BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
+BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
+PATH=`%(testbase)s/Scripts/get-bro-env path`
+TZ=UTC
+LC_ALL=C
+TRACES=%(testbase)s/Traces
+TMPDIR=%(testbase)s/.tmp
+BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
+TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env brobase`/testing/scripts/diff-canonifier

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
new file mode 100644
index 0000000..4e8dd6a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# @TEST-EXEC: bro -NN Bro::Kafka >output
+# @TEST-EXEC: btest-diff output

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/.gitignore b/metron-sensors/packet-capture/.gitignore
new file mode 100644
index 0000000..2efc4c0
--- /dev/null
+++ b/metron-sensors/packet-capture/.gitignore
@@ -0,0 +1,3 @@
+roles
+.vagrant
+*.retry



Mime
View raw message