metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [1/5] incubator-metron git commit: METRON-659 Emulate Sensors in Development Environments (nickwallen) closes apache/incubator-metron#417
Date Mon, 23 Jan 2017 15:12:18 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 368e7ad63 -> d1fcda604


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/tasks/main.yml b/metron-deployment/roles/sensor-stubs/tasks/main.yml
new file mode 100644
index 0000000..708880f
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/tasks/main.yml
@@ -0,0 +1,40 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create home directory
+  file: path={{ item }} state=directory mode=0755
+  with_items:
+    - "{{ sensor_stubs_home }}"
+    - "{{ sensor_stubs_bin }}"
+    - "{{ sensor_stubs_data }}"
+  
+- name: Distribute stub data
+  copy: src={{ item }} dest={{ sensor_stubs_data }}/
+  with_items:
+    - bro.out
+    - snort.out
+    - yaf.out
+
+- name: Install service script
+  template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
+
+- name: Install sensor stubs
+  template: src={{ item }} dest={{ sensor_stubs_bin }}/ mode=0755
+  with_items:
+    - start-bro-stub
+    - start-snort-stub
+    - start-yaf-stub
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/sensor-stubs b/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
new file mode 100644
index 0000000..eaab69d
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron sensor-stubs service
+# chkconfig: 345 20 80
+# description: Simulates the behavior of a sensor by sending canned telemetry data to a Kafka
topic
+# processname: sensor-stubs
+#
+NAME=sensor-stubs
+DESC="Simulates the behavior of a sensor by sending canned telemetry data to a Kafka topic"
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="{{ sensor_stubs_log }}"
+DAEMON_PATH="{{ sensor_stubs_home }}"
+DATA="{{ sensor_stubs_data }}"
+
+# the delay between each 'batch' of messages in seconds.  
+# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
+DELAY={{ sensor_stubs_delay }}
+
+# the number of messages to send in each batch.  
+# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
+COUNT={{ sensor_stubs_count }}
+
+#
+# which sensors? defaults to bro, snort, yaf
+#
+SENSORS="${@:2}"
+if [ -z "${SENSORS}" ]; then 
+    SENSORS=('bro' 'yaf' 'snort')
+fi
+
+#
+# start a sensor stub
+#
+start() {  
+
+  # if pidfile exists, do not start another
+  PIDFILE="/var/run/$NAME-$1.pid"
+  if [ -f $PIDFILE ]; then
+      PID=`cat $PIDFILE`
+      printf "%10s: %s \n" "$1" "OK [$PID]"
+      return
+  fi
+
+  # kick-off the daemon
+  DAEMON="{{ sensor_stubs_bin }}/start-$1-stub $DELAY $COUNT"
+  PID=`$DAEMON >> $LOGFILE 2>&1 & echo $!`
+
+  if [ -z $PID ]; then
+      printf "%10s: %s \n" "$1" "Fail"
+  else
+      echo $PID > $PIDFILE
+      printf "%10s: %s \n" "$1" "Ok [$PID]"
+  fi
+}
+
+#
+# stop a sensor stub
+#
+stop() {
+  PIDFILE="/var/run/$NAME-$1.pid"
+  cd $DAEMON_PATH
+  if [ -f $PIDFILE ]; then
+      PID=`cat $PIDFILE`
+      while sleep 1
+        echo -n "."
+        kill -0 $PID >/dev/null 2>&1
+      do
+        kill $PID
+      done
+
+      printf "%10s: %s \n" "$1" "Stopped [$PID]"
+      rm -f $PIDFILE
+  else
+      printf "%10s: %s \n" "$1" "Not running"
+  fi
+}
+
+#
+# status check of sensor stub
+#
+status() {
+  PIDFILE="/var/run/$NAME-$1.pid"
+  if [ -f $PIDFILE ]; then
+    PID=`cat $PIDFILE`
+    if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+      printf "%10s: %s \n" "$1" "Process dead but pidfile exists"
+    else
+      printf "%10s: %s \n" "$1" "Running [${PID}]"
+    fi
+  else
+    printf "%10s: %s \n" "$1" "Not running"
+  fi
+}
+
+case "$1" in
+
+  ##############################################################################
+  # start
+  #
+  start)
+    printf "%-50s \n" "Starting $NAME..."
+    for sensor in "${SENSORS[@]}"; do
+      start $sensor
+    done
+  ;;
+
+  ##############################################################################
+  # status
+  #
+  status)
+    printf "%-50s \n" "Checking $NAME..."
+    for sensor in "${SENSORS[@]}"; do
+      status $sensor
+    done
+  ;;
+
+  ##############################################################################
+  # stop
+  #
+  stop)
+    printf "%-50s \n" "Stopping $NAME..."
+    for sensor in "${SENSORS[@]}"; do
+      stop $sensor
+    done
+  ;;
+
+  ##############################################################################
+  # restart
+  #
+  restart)
+    $0 stop
+    $0 start
+  ;;
+
+  *)
+    echo "Usage: $0 {status|start|stop|restart}"
+    exit 1
+esac

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
new file mode 100644
index 0000000..fb70d1e
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in 
+# batches.  the timestamp of the message is altered to match current
+# system time.  the number of messages sent in each batch, along with 
+# the time delay between batches can be configured.
+#
+# start-bro-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'.  the messages are drawn randomly
+# from the entire set of canned data. 
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+INPUT="{{ sensor_stubs_data }}/bro.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="bro"
+
+while true; do
+  
+  # transform the bro timestamp and push to kafka
+  SEARCH="\"ts\"\:[0-9]\+.[0-9]\{6\}"
+  REPLACE="\"ts\"\:`date +%s`.000000"
+  shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url
}} --topic $TOPIC
+  
+  sleep $DELAY
+done

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
new file mode 100644
index 0000000..3123782
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in 
+# batches.  the timestamp of the message is altered to match current
+# system time.  the number of messages sent in each batch, along with 
+# the time delay between batches can be configured.
+#
+# start-snort-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'.  the messages are drawn randomly
+# from the entire set of canned data. 
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+INPUT="{{ sensor_stubs_data }}/snort.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="snort"
+
+while true; do
+  
+  # transform the timestamp and push to kafka
+  SEARCH="[^,]\+ ,"
+  REPLACE="`date +'%m\/%d\/%y-%H:%M:%S'`.000000 ,"
+  shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url
}} --topic $TOPIC
+  
+  sleep $DELAY
+done

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
new file mode 100644
index 0000000..1966d39
--- /dev/null
+++ b/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in 
+# batches.  the timestamp of the message is altered to match current
+# system time.  the number of messages sent in each batch, along with 
+# the time delay between batches can be configured.
+#
+# start-yaf-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'.  the messages are drawn randomly
+# from the entire set of canned data. 
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+
+INPUT="{{ sensor_stubs_data }}/yaf.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="yaf"
+
+while true; do
+  
+  # transform the timestamp and push to kafka
+  SEARCH="[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\.[0-9]\+"
+  REPLACE="`date +'%Y-%m-%d %H:%M:%S'`.000"
+  shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url
}} --topic $TOPIC
+  
+  sleep $DELAY
+done

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/vagrant/full-dev-platform/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile
index 1c6e3f0..c73a706 100644
--- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile
+++ b/metron-deployment/vagrant/full-dev-platform/Vagrantfile
@@ -17,7 +17,7 @@
 require 'getoptlong'
 
 ansibleTags=''
-ansibleSkipTags='solr,yaf'
+ansibleSkipTags='solr,sensors'
 
 begin
    opts = GetoptLong.new(

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
index 3963ff4..28c73da 100644
--- a/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
+++ b/metron-deployment/vagrant/quick-dev-platform/Vagrantfile
@@ -17,7 +17,7 @@
 require 'getoptlong'
 
 ansibleTags='ambari-agent,hdp-deploy,metron'
-ansibleSkipTags='solr,yaf'
+ansibleSkipTags='solr,sensors'
 
 begin
    opts = GetoptLong.new(

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d1fcda60/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8b3cbee..215333b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -268,6 +268,7 @@
                         <exclude>**/hbase/data/**</exclude>
                         <exclude>**/kafkazk/data/**</exclude>
                         <exclude>**/wait-for-it.sh</exclude>
+			<exclude>**/*.out</exclude>
                     </excludes>
                 </configuration>
             </plugin>


Mime
View raw message