metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [5/5] incubator-metron git commit: METRON-659 Emulate Sensors in Development Environments (nickwallen) closes apache/incubator-metron#417
Date Mon, 23 Jan 2017 15:12:22 GMT
METRON-659 Emulate Sensors in Development Environments (nickwallen) closes apache/incubator-metron#417


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/d1fcda60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/d1fcda60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/d1fcda60

Branch: refs/heads/master
Commit: d1fcda6043fe608167cde616214e4059663fb131
Parents: 368e7ad
Author: nickwallen <nick@nickallen.org>
Authored: Mon Jan 23 10:11:42 2017 -0500
Committer: Nick Allen <nick@nickallen.org>
Committed: Mon Jan 23 10:11:42 2017 -0500

----------------------------------------------------------------------
 metron-deployment/amazon-ec2/run.sh             |     2 +-
 metron-deployment/playbooks/metron_install.yml  |    12 +
 metron-deployment/roles/monit/defaults/main.yml |     4 +-
 metron-deployment/roles/monit/tasks/main.yml    |     8 +
 .../roles/monit/tasks/monit-definitions.yml     |    18 +-
 .../monit/tasks/monit-sensor-definitions.yml    |    38 +
 .../monit/tasks/monit-stub-definitions.yml      |    30 +
 .../roles/monit/templates/monit/bro-stub.monit  |    25 +
 .../monit/templates/monit/snort-stub.monit      |    25 +
 .../roles/monit/templates/monit/yaf-stub.monit  |    25 +
 metron-deployment/roles/sensor-stubs/README.md  |    85 +
 .../roles/sensor-stubs/defaults/main.yml        |    25 +
 .../roles/sensor-stubs/files/bro.out            |  1346 +
 .../roles/sensor-stubs/files/snort.out          | 27404 +++++++++++++++++
 .../roles/sensor-stubs/files/yaf.out            | 22164 +++++++++++++
 .../roles/sensor-stubs/tasks/main.yml           |    40 +
 .../roles/sensor-stubs/templates/sensor-stubs   |   154 +
 .../roles/sensor-stubs/templates/start-bro-stub |    55 +
 .../sensor-stubs/templates/start-snort-stub     |    55 +
 .../roles/sensor-stubs/templates/start-yaf-stub |    56 +
 .../vagrant/full-dev-platform/Vagrantfile       |     2 +-
 .../vagrant/quick-dev-platform/Vagrantfile      |     2 +-
 pom.xml                                         |     1 +
 23 files changed, 51555 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/amazon-ec2/run.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh
index fac3f68..cad4fb7 100755
--- a/metron-deployment/amazon-ec2/run.sh
+++ b/metron-deployment/amazon-ec2/run.sh
@@ -67,6 +67,6 @@ RC=$?; if [[ $RC != 0 ]]; then exit $RC; fi
 cd $DEPLOYDIR
 export EC2_INI_PATH=conf/ec2.ini
 ansible-playbook -i ec2.py playbook.yml \
-  --skip-tags="solr" \
+  --skip-tags="solr, sensor-stubs" \
   --extra-vars="env=$ENV" \
   $EXTRA_ARGS

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index a18fb30..9f2d231 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -137,6 +137,18 @@
     - sensors
 
 #
+# sensor stubs - rather than deploying the sensors, deploy ligher weight
+# stubs that mimic the behavior of the real sensors
+#
+- hosts: sensors
+  become: true
+  roles:
+    - { role: ambari_gather_facts }
+    - { role: sensor-stubs }
+  tags:
+    - sensor-stubs
+
+#
 # monitor and start metron services with monit
 #
 - hosts: metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml
index b1818bc..608be38 100644
--- a/metron-deployment/roles/monit/defaults/main.yml
+++ b/metron-deployment/roles/monit/defaults/main.yml
@@ -19,8 +19,8 @@ monit_home: /usr/local/monit
 monit_config_home: /etc/monit.d
 monit_user: admin
 monit_pass: monit
-topology_start_timeout: 60
-topology_stop_timeout: 60
+topology_start_timeout: 120
+topology_stop_timeout: 120
 
 bro_pid_file: /usr/local/bro/spool/bro/.pid
 elasticsearch_pid_file: /var/run/elasticsearch/elasticsearch.pid

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/main.yml b/metron-deployment/roles/monit/tasks/main.yml
index ecae787..3718797 100644
--- a/metron-deployment/roles/monit/tasks/main.yml
+++ b/metron-deployment/roles/monit/tasks/main.yml
@@ -19,3 +19,11 @@
 - include: monit.yml
 - include: scripts.yml
 - include: monit-definitions.yml
+
+- include: monit-sensor-definitions.yml
+  tags:
+    - sensors
+
+- include: monit-stub-definitions.yml
+  tags:
+    - sensor-stubs
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index f9e3897..1d29795 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -16,10 +16,6 @@
 #  limitations under the License.
 #
 ---
-- name: Create monit definition for bro
-  template: src=monit/bro.monit dest={{ monit_config_home }}/bro.monit
-  when: ("sensors" in group_names) and (install_bro | default(True))
-
 - name: Create monit definition for elasticsearch
   template: src=monit/elasticsearch.monit dest={{ monit_config_home }}/elasticsearch.monit
   when: ("search" in group_names) and (install_elasticsearch | default(True))
@@ -47,19 +43,9 @@
 - name: Create monit definition for pcap-replay
   template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
   when: ("sensors" in group_names) and (install_pcap_replay | default(False))
-
+  tags: sensors
+  
 - name: Create monit definition for pcap-service
   template: src=monit/pcap-service.monit dest={{ monit_config_home }}/pcap-service.monit
   when: ("pcap_server" in group_names)
 
-- name: Create monit definition for pycapa
-  template: src=monit/pycapa.monit dest={{ monit_config_home }}/pycapa.monit
-  when: ("sensors" in group_names) and (install_pycapa | default(True))
-
-- name: Create monit definition for snort
-  template: src=monit/snort.monit dest={{ monit_config_home }}/snort.monit
-  when: ("sensors" in group_names) and (install_snort | default(True))
-
-- name: Create monit definition for yaf
-  template: src=monit/yaf.monit dest={{ monit_config_home }}/yaf.monit
-  when: ("sensors" in group_names) and (install_yaf | default(True))

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml b/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
new file mode 100644
index 0000000..4b2d5de
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
@@ -0,0 +1,38 @@
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create monit definition for pcap-replay
+  template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
+  when: ("sensors" in group_names) and (install_pcap_replay | default(False))
+
+- name: Create monit definition for pycapa
+  template: src=monit/pycapa.monit dest={{ monit_config_home }}/pycapa.monit
+  when: ("sensors" in group_names) and (install_pycapa | default(True))
+
+- name: Create monit definition for snort
+  template: src=monit/snort.monit dest={{ monit_config_home }}/snort.monit
+  when: ("sensors" in group_names) and (install_snort | default(True))
+
+- name: Create monit definition for yaf
+  template: src=monit/yaf.monit dest={{ monit_config_home }}/yaf.monit
+  when: ("sensors" in group_names) and (install_yaf | default(True))
+  
+- name: Create monit definition for bro
+  template: src=monit/bro.monit dest={{ monit_config_home }}/bro.monit
+  when: ("sensors" in group_names) and (install_bro | default(True))
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml b/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
new file mode 100644
index 0000000..fde711a
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
@@ -0,0 +1,30 @@
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create monit definition for snort
+  template: src=monit/snort-stub.monit dest={{ monit_config_home }}/snort.monit
+  when: ("sensors" in group_names) and (install_snort | default(True))
+
+- name: Create monit definition for yaf
+  template: src=monit/yaf-stub.monit dest={{ monit_config_home }}/yaf.monit
+  when: ("sensors" in group_names) and (install_yaf | default(True))
+  
+- name: Create monit definition for bro
+  template: src=monit/bro-stub.monit dest={{ monit_config_home }}/bro.monit
+  when: ("sensors" in group_names) and (install_bro | default(True))
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/templates/monit/bro-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/bro-stub.monit b/metron-deployment/roles/monit/templates/monit/bro-stub.monit
new file mode 100644
index 0000000..54bdbd6
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/bro-stub.monit
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+check process bro
+  with pidfile /var/run/sensor-stubs-bro.pid
+  start program = "/etc/init.d/sensor-stubs start bro"
+  stop program = "/etc/init.d/sensor-stubs stop bro"
+  if does not exist then restart
+  group bro
+  group sensors
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/templates/monit/snort-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/snort-stub.monit b/metron-deployment/roles/monit/templates/monit/snort-stub.monit
new file mode 100644
index 0000000..b782690
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/snort-stub.monit
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+check process snort
+  with pidfile /var/run/sensor-stubs-snort.pid
+  start program = "/etc/init.d/sensor-stubs start snort"
+  stop program = "/etc/init.d/sensor-stubs stop snort"
+  if does not exist then restart
+  group snort
+  group sensors
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/yaf-stub.monit b/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
new file mode 100644
index 0000000..2a92a53
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+check process yaf
+  with pidfile /var/run/sensor-stubs-yaf.pid
+  start program = "/etc/init.d/sensor-stubs start yaf"
+  stop program = "/etc/init.d/sensor-stubs stop yaf"
+  if does not exist then restart
+  group yaf
+  group sensors
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/README.md b/metron-deployment/roles/sensor-stubs/README.md
new file mode 100644
index 0000000..be9b7af
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/README.md
@@ -0,0 +1,85 @@
+
+# Sensor Stubs
+
+A service has been created to simulate the behavior of a sensor by sending canned telemetry
data to a Kafka topic.  These "Sensor Stubs" consume fewer resources than the actual sensor
that they replace.
+ 
+### (Q) How do the sensor stubs work?
+
+The stubs are installed with a set of canned data for each sensor type; Bro, Snort and YAF.
 A subset of this canned data is randomly selected and sent to the Kafka topic in batches.
 The timestamp of each message is updated to match current system time.  
+
+### (Q) How do I configure the message rate?
+
+The number of telemetry messages sent in each batch, along with the time delay between batches
is configurable.  Before installation, these values can be configured by redefining `sensor_stubs_delay`
and `sensor_stubs_count`.  The values can also be configured by altering the deployed system
service script at `/etc/init.d/sensor-stubs`.
+
+### (Q) How do I install the sensor stubs?
+
+Using the default playbooks, this role can be installed by using the Ansible tag `sensor-stubs`.
 This service is installed on the same hosts where the sensors would be; defined by the `sensors`
host group.
+
+The defaults for the "Quick Dev" and "Full Dev" environments have been changed so that  the
Sensor Stubs are installed by default, rather than the sensors themselves.  The Amazon EC2
environment continues to install the original sensors by default.
+
+### (Q) How do I use the sensor stubs?
+
+Start all sensor stubs.  The output includes the PID for each running sensor stub.
+```
+$ service sensor-stubs start
+Starting sensor-stubs...
+       bro: Ok [26505]
+       yaf: Ok [26507]
+     snort: Ok [26509]
+```
+
+Check the status of each sensor stub.
+```
+$ service sensor-stubs status
+Checking sensor-stubs...
+       bro: Running [26505]
+       yaf: Running [26507]
+     snort: Running [26509]
+```
+
+Stop all sensor stubs.
+```
+$ service sensor-stubs stop
+Stopping sensor-stubs...
+..       bro: Ok [26505]
+..       yaf: Ok [26507]
+..     snort: Ok [26509]
+```
+
+Check the status.  All sensor stubs should be stopped.
+```
+$ service sensor-stubs status
+Checking sensor-stubs...
+       bro: Not running
+       yaf: Not running
+     snort: Not running
+```
+
+Start only the Bro sensor stub.
+```
+$ service sensor-stubs start bro
+Starting sensor-stubs...
+       bro: OK [11616]
+```
+
+Stop the Bro sensor stub.
+```
+$ service sensor-stubs stop bro
+Stopping sensor-stubs...
+..       bro: Ok [11616]
+```
+
+### (Q) How do I install the original sensors?
+
+The default behavior can be changed by skipping the `sensor-stubs` flag and including the
`sensors` flag. For example, to deploy "Quick Dev" with the original sensors run the following
command.
+
+```
+cd metron-deployment/vagrant/quick-dev-platform
+vagrant --ansible-skip-tags="sensor-stubs,solr" up
+```
+
+### (Q) Where does the mock data come from?
+
+The data produced by the sensor stubs was generated by running the sensors against the example
pcap file that is distributed with Metron.  This ensures that the data produced by the sensor
stubs is similar to the data produced when using the actual sensors.
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/defaults/main.yml b/metron-deployment/roles/sensor-stubs/defaults/main.yml
new file mode 100644
index 0000000..e8efb9e
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/defaults/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+sensor_stubs_home: /opt/sensor-stubs
+sensor_stubs_bin: "{{ sensor_stubs_home }}/bin"
+sensor_stubs_data: "{{ sensor_stubs_home }}/data"
+sensor_stubs_log: /var/log/sensor-stubs.log
+
+sensor_stubs_delay: 2
+sensor_stubs_count: 10
+kafka_home: /usr/hdp/current/kafka-broker
\ No newline at end of file


Mime
View raw message