metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [05/30] incubator-metron git commit: Using an older forked version of Bro that has a Kafka writer to publish Bro data
Date Fri, 29 Jan 2016 20:04:18 GMT
Using an older forked version of Bro that has a Kafka writer to publish Bro data


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/dd5b60bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/dd5b60bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/dd5b60bb

Branch: refs/heads/master
Commit: dd5b60bbc12f23e95c38557a915a0f73b00f47d4
Parents: dc7f8a8
Author: Nick Allen <nick@nickallen.org>
Authored: Mon Jan 25 14:35:10 2016 -0500
Committer: Nick Allen <nick@nickallen.org>
Committed: Mon Jan 25 14:35:10 2016 -0500

----------------------------------------------------------------------
 .../inventory/metron_example/group_vars/all     |   6 +-
 .../inventory/singlenode-vagrant/group_vars/all |   6 +-
 .../bro/files/Bro-2.2-344-Linux-x86_64.rpm      | Bin 0 -> 2914676 bytes
 .../roles/bro/files/flume-bro.conf.template     |  28 ----------
 .../librdkafka-devel-0.9.1-10.el6.x86_64.rpm    | Bin 0 -> 237172 bytes
 .../files/librdkafka1-0.9.1-10.el6.x86_64.rpm   | Bin 0 -> 196580 bytes
 deployment/roles/bro/tasks/bro.yml              |  51 -----------------
 deployment/roles/bro/tasks/flume.yml            |  41 --------------
 deployment/roles/bro/tasks/main.yml             |  56 ++++++++++++++++---
 deployment/roles/bro/vars/main.yml              |   5 +-
 10 files changed, 56 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index 12515c7..3ecfe61 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -21,9 +21,7 @@ daq_version: "2.0.6-1"
 iface: "eth0"
 yaf_topic: "ipfix"
 snort_topic: "snort"
-bro_conn_topic: "bro-conn"
-bro_dns_topic: "bro-dns"
-bro_software_topic: "bro-software"
+bro_topic: "bro"
 pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
 pycapa_home: "/opt/pycapa"
-pycapa_topic: "pcap"
\ No newline at end of file
+pycapa_topic: "pcap"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 79f1794..cf78381 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -24,9 +24,7 @@ daq_version: "2.0.6-1"
 iface: "eth0"
 yaf_topic: "ipfix"
 snort_topic: "snort"
-bro_conn_topic: "bro-conn"
-bro_dns_topic: "bro-dns"
-bro_software_topic: "bro-software"
+bro_topic: "bro"
 pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
 pycapa_home: "/opt/pycapa"
-pycapa_topic: "pcap"
\ No newline at end of file
+pycapa_topic: "pcap"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/files/Bro-2.2-344-Linux-x86_64.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/files/Bro-2.2-344-Linux-x86_64.rpm b/deployment/roles/bro/files/Bro-2.2-344-Linux-x86_64.rpm
new file mode 100644
index 0000000..745a9e9
Binary files /dev/null and b/deployment/roles/bro/files/Bro-2.2-344-Linux-x86_64.rpm differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/files/flume-bro.conf.template
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/files/flume-bro.conf.template b/deployment/roles/bro/files/flume-bro.conf.template
deleted file mode 100644
index 0881e22..0000000
--- a/deployment/roles/bro/files/flume-bro.conf.template
+++ /dev/null
@@ -1,28 +0,0 @@
-
-bro-conn.sources = exec-source
-bro-conn.channels = memory-channel
-bro-conn.sinks = kafka-sink logger-sink
-
-# bro data is logged to a set of files
-bro-conn.sources.exec-source.type = exec
-bro-conn.sources.exec-source.command = tail -F /usr/local/bro/logs/current/conn.log
-bro-conn.sources.exec-source.restart = true
-bro-conn.sources.exec-source.logStdErr = true
-
-# bro alerts are sent to kafka
-bro-conn.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
-bro-conn.sinks.kafka-sink.brokerList = localhost:9092
-bro-conn.sinks.kafka-sink.topic = bro-conn
-
-# also log events
-bro-conn.sinks.logger-sink.type = logger
-
-# buffer events in memory
-bro-conn.channels.memory-channel.type = memory
-bro-conn.channels.memory-channel.capacity = 1000
-bro-conn.channels.memory-channel.transactionCapacity = 100
-
-# bind the source and sink to the channel
-bro-conn.sources.exec-source.channels = memory-channel
-bro-conn.sinks.kafka-sink.channel = memory-channel
-bro-conn.sinks.logger-sink.channel = memory-channel

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/files/librdkafka-devel-0.9.1-10.el6.x86_64.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/files/librdkafka-devel-0.9.1-10.el6.x86_64.rpm b/deployment/roles/bro/files/librdkafka-devel-0.9.1-10.el6.x86_64.rpm
new file mode 100644
index 0000000..5cd127c
Binary files /dev/null and b/deployment/roles/bro/files/librdkafka-devel-0.9.1-10.el6.x86_64.rpm
differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/files/librdkafka1-0.9.1-10.el6.x86_64.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/files/librdkafka1-0.9.1-10.el6.x86_64.rpm b/deployment/roles/bro/files/librdkafka1-0.9.1-10.el6.x86_64.rpm
new file mode 100644
index 0000000..117ba77
Binary files /dev/null and b/deployment/roles/bro/files/librdkafka1-0.9.1-10.el6.x86_64.rpm
differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/bro.yml b/deployment/roles/bro/tasks/bro.yml
deleted file mode 100644
index 836ff90..0000000
--- a/deployment/roles/bro/tasks/bro.yml
+++ /dev/null
@@ -1,51 +0,0 @@
----
-- name: Install Bro dependencies
-  yum: name={{item}} state=latest update_cache=no
-  with_items:
-    - cmake
-    - make
-    - gcc
-    - gcc-c++
-    - flex
-    - bison
-    - libpcap-devel
-    - openssl-devel
-    - python-devel
-    - swig
-    - zlib-devel
-
-- name: Install geoip data
-  yum: name={{item}} state=latest update_cache=yes
-  with_items:
-    - epel-release
-    - GeoIP
-    - GeoIP-devel
-    - GeoIP-data
-
-- name: Download bro
-  get_url:
-    url: "https://www.bro.org/downloads/release/bro-{{bro_version}}.tar.gz"
-    dest: "/tmp/bro-{{bro_version}}.tar.gz"
-
-- name: Extract tarball
-  unarchive:
-    src: "/tmp/bro-{{bro_version}}.tar.gz"
-    dest: /tmp
-    copy: no
-    creates: "/tmp/bro-{{bro_version}}"
-
-- name: Build and install bro
-  shell: "{{item}}"
-  args:
-    chdir: "/tmp/bro-{{bro_version}}"
-    creates: /usr/local/bro/bin/broctl
-  with_items:
-    - ./configure
-    - make
-    - make install
-
-- name: Configure json logging
-  lineinfile: dest=/usr/local/bro/share/bro/site/local.bro line="@load policy/tuning/json-logs"
-
-- name: Install updated bro configuration
-  shell: /usr/local/bro/bin/broctl install

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/tasks/flume.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/flume.yml b/deployment/roles/bro/tasks/flume.yml
deleted file mode 100644
index 4f8721f..0000000
--- a/deployment/roles/bro/tasks/flume.yml
+++ /dev/null
@@ -1,41 +0,0 @@
----
-- name: Install flume configurations
-  copy: src={{ item.src }} dest={{ item.dest }}
-  with_items:
-    - { src: flume-bro.conf.template, dest: /etc/flume/conf/flume-bro-conn.conf }
-    - { src: flume-bro.conf.template, dest: /etc/flume/conf/flume-bro-dns.conf }
-    - { src: flume-bro.conf.template, dest: /etc/flume/conf/flume-bro-software.conf }
-
-- name: Set kafka broker in flume
-  lineinfile: >
-    dest="{{ item.dest }}"
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
-  with_items:
-    - { dest: /etc/flume/conf/flume-bro-conn.conf,
-        regexp: '^bro-conn\.sinks\.kafka-sink\.brokerList.*$',
-        line: 'bro-conn.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
-    - { dest: /etc/flume/conf/flume-bro-dns.conf,
-        regexp: '^bro-dns\.sinks\.kafka-sink\.brokerList.*$',
-        line: 'bro-dns.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
-    - { dest: /etc/flume/conf/flume-bro-software.conf,
-        regexp: '^bro-software\.sinks\.kafka-sink\.brokerList.*$',
-        line: 'bro-software.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
-
-- name: Set kafka topics in flume
-  lineinfile: >
-    dest="{{ item.dest }}"
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
-  with_items:
-    - { dest: /etc/flume/conf/flume-bro-conn.conf,
-        regexp: '^bro-conn\.sinks\.kafka-sink\.topic.*$',
-        line: 'bro-conn.sinks.kafka-sink.topic = {{ bro_conn_topic }}' }
-    - { dest: /etc/flume/conf/flume-bro-dns.conf,
-        regexp: '^bro-dns\.sinks\.kafka-sink\.topic.*$',
-        line: 'bro-dns.sinks.kafka-sink.topic = {{ bro_dns_topic }}' }
-    - { dest: /etc/flume/conf/flume-bro-software.conf,
-        regexp: '^bro-software\.sinks\.kafka-sink\.topic.*$',
-        line: 'bro-software.sinks.kafka-sink.topic = {{ bro_software_topic }}' }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/main.yml b/deployment/roles/bro/tasks/main.yml
index 1585cfe..4d1e8fe 100644
--- a/deployment/roles/bro/tasks/main.yml
+++ b/deployment/roles/bro/tasks/main.yml
@@ -1,14 +1,54 @@
 ---
-- include: bro.yml
+- name: Install EPEL repository
+  yum: name=epel-release
 
-- include: flume.yml
+- name: Install prerequisites
+  yum: name={{ item }}
+  with_items:
+    - libselinux-python
+    - libpcap
+    - libpcap-devel
+
+- name: Install geoip data
+  yum: name={{ item }}
+  with_items:
+    - GeoIP
+    - GeoIP-devel
+    - GeoIP-data
+
+- name: Copy over RPM dependencies
+  copy: src={{ item }} dest=/tmp
+  with_items: rpms
+
+- name: Install Bro and dependencies
+  shell: rpm -i /tmp/{{ item }}
+  with_items: rpms
 
-- name: Start flume to consume bro data
-  service: name=flume-agent state=started args={{ item }}
+- name: Configure bro outputs
+  lineinfile:
+    dest: /opt/bro/share/bro/site/local.bro
+    line: "{{ item }}"
   with_items:
-    - bro-conn
-    - bro-dns
-    - bro-software
+    - "@load policy/tuning/json-logs"
+    - "@load policy/tuning/logs-to-kafka"
+
+- name: Configure kafka broker
+  lineinfile:
+    dest: /opt/bro/share/bro/base/frameworks/logging/writers/kafka.bro
+    regexp: '^.*server_list.*$'
+    line: '\tconst server_list = "{{ kafka_broker_url }}" &redef;'
+
+- name: Configure kafka topic
+  lineinfile:
+    dest: /opt/bro/share/bro/base/frameworks/logging/writers/kafka.bro
+    regexp: '^.*topic_name.*$'
+    line: '\tconst topic_name = "{{ bro_topic }}" &redef;'
+
+- name: Remove funky broctl plugins
+  shell: rm -rf /opt/bro/lib/broctl/plugins/._*
+
+- name: Install updated bro configuration
+  shell: /opt/bro/bin/broctl install
 
 - name: Start bro
-  shell: /usr/local/bro/bin/broctl deploy
+  shell: /opt/bro/bin/broctl start

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/dd5b60bb/deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/vars/main.yml b/deployment/roles/bro/vars/main.yml
index dddd281..1e97fe1 100644
--- a/deployment/roles/bro/vars/main.yml
+++ b/deployment/roles/bro/vars/main.yml
@@ -1,2 +1,5 @@
 ---
-bro_version: 2.4.1
+rpms:
+  - librdkafka1-0.9.1-10.el6.x86_64.rpm
+  - librdkafka-devel-0.9.1-10.el6.x86_64.rpm
+  - Bro-2.2-344-Linux-x86_64.rpm


Mime
View raw message