eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hdenduk...@apache.org
Subject incubator-eagle git commit: EAGLE-154 hadoop-metric monitoring sandbox starter script. - hadoop-metric monitoring sandbox starter script and policy creation. - update file access permission 755 to hadoop-metric-init.sh file. Added Eagle home path
Date Fri, 19 Feb 2016 01:03:53 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 88d0126cb -> fba3aebae


EAGLE-154 hadoop-metric monitoring sandbox starter script.
   - hadoop-metric monitoring sandbox starter script and policy creation.
   -  update file access permission 755 to hadoop-metric-init.sh file. Added Eagle home path
${EAGLE_HOME} to locate sandbox-hadoopjmx-topology.conf in hadoop-metric-monitoring.sh.

https://issues.apache.org/jira/browse/EAGLE-154

Author: @hdendukuri <hdendukuri@ebay.com>
Reviewer: @yonzhang <yonzhang@ebay.com>

Closes https://github.com/apache/incubator-eagle/pull/87


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fba3aeba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fba3aeba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fba3aeba

Branch: refs/heads/master
Commit: fba3aebae232ba4090dc3fdc64dbdd2946b197ce
Parents: 88d0126
Author: hdendukuri <hdendukuri@ebay.com>
Authored: Wed Feb 17 11:32:26 2016 -0800
Committer: hdendukuri <hdendukuri@ebay.com>
Committed: Wed Feb 17 11:32:26 2016 -0800

----------------------------------------------------------------------
 .../src/main/bin/hadoop-metric-monitor.sh       |   6 +-
 .../examples/hadoop-metric-policy-create.sh     |  45 +++++++
 .../examples/hadoop-metric-sandbox-starter.sh   | 125 +++++++++++++++++++
 .../src/main/resources/hadoop-metric-init.sh    |   0
 4 files changed, 173 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
index 7205ae3..327bec1 100644
--- a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
+++ b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
@@ -34,7 +34,7 @@ if [ "$active" ]; then
  ${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology stop
 fi
 echo "start Eagle Hadoop Metric Monitoring topology"
-${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor
--topology sandbox-hadoopjmx-topology --config ../conf/sandbox-hadoopjmx-topology.conf start
+${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor
--topology sandbox-hadoopjmx-topology --config ${EAGLE_HOME}/conf/sandbox-hadoopjmx-topology.conf
start
 
 ######################################################################
 ##            Setup minutely crontab job for HADOOP METRIC
@@ -43,8 +43,8 @@ echo "set up crontab script"
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cp $DIR/../tools/hadoop_jmx_collector/config-sample.json $DIR/../tools/hadoop_jmx_collector/config.json
 command="python $DIR/../tools/hadoop_jmx_collector/hadoop_jmx_kafka.py"
-echo $job
 job="* * * * * $command >> $DIR/../logs/hadoop_metric.log"
+echo "$job"
 cat <(fgrep -i -v "$command" <(crontab -l)) <(echo "$job") | crontab -
 
-echo "$(crontab -l)"
\ No newline at end of file
+echo "$(crontab -l)"

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh b/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
new file mode 100644
index 0000000..fc0b639
--- /dev/null
+++ b/eagle-assembly/src/main/examples/hadoop-metric-policy-create.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with`
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname $0)/../bin/eagle-env.sh
+
+##### add policies ##########
+echo ""
+echo "Importing policy: safeModePolicy "
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
\
+ "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDefinitionService"
\
+ -d '
+ [
+     {
+       "prefix": "alertdef",
+       "tags": {
+         "site": "sandbox",
+         "dataSource": "hadoopJmxMetricDataSource",
+         "policyId": "safeModePolicy",
+         "alertExecutorId": "hadoopJmxMetricAlertExecutor",
+         "policyType": "siddhiCEPEngine"
+       },
+       "description": "jmx metric ",
+       "policyDef": "{\"expression\":\"from hadoopJmxMetricEventStream[component==\\\"namenode\\\"
and metric == \\\"hadoop.namenode.fsnamesystemstate.fsstate\\\" and convert(value, \\\"long\\\")
> 0]#window.externalTime(timestamp ,10 min) select metric, host, value, timestamp, component,
site insert into tmp; \",\"type\":\"siddhiCEPEngine\"}",
+       "enabled": true,
+       "dedupeDef": "{\"alertDedupIntervalMin\":10,\"emailDedupIntervalMin\":10}",
+       "notificationDef": "[{\"sender\":\"eagle@apache.org\",\"recipients\":\"eagle@apache.org\",\"subject\":\"missing
block found.\",\"flavor\":\"email\",\"id\":\"email_1\",\"tplFileName\":\"\"}]"
+     }
+ ]
+ '
+
+exit 0
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh b/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
new file mode 100644
index 0000000..8f227b0
--- /dev/null
+++ b/eagle-assembly/src/main/examples/hadoop-metric-sandbox-starter.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname $0)/../bin/eagle-env.sh
+eagle_bin=$EAGLE_HOME/bin
+
+
+###################################################################
+#             STEP 1: Check Installation
+###################################################################
+
+echo "STEP [1/3]: checking environment"
+$eagle_bin/eagle-check-env.sh
+[ $? != 0 ] && exit 1
+
+pid_dir=/var/run
+
+# Check HBase if it has been started
+hbase_master_pid=${pid_dir}/hbase/hbase-hbase-master.pid
+hbase_regionserver_pid=${pid_dir}/hbase/hbase-hbase-regionserver.pid
+echo "Checking if hbase is running ..."
+
+if [ -f $hbase_master_pid ] && \
+	ps aux | grep -v grep | grep $(cat $hbase_master_pid) > /dev/null
+then
+	echo "HBase Master is running as process `cat $hbase_master_pid`."
+else
+	echo "Error: HBase Master is not running. Please start it via Ambari."
+	exit 1
+fi
+
+if [ -f $hbase_regionserver_pid ] && \
+	ps aux | grep -v grep | grep $(cat $hbase_regionserver_pid) > /dev/null
+then
+	echo "HBase RegionServer is running as process `cat $hbase_regionserver_pid`."
+else
+	echo "Error: HBase RegionServer is not running. Please start it via Ambari."
+	exit 1
+fi
+
+# Check kafka if it has been started
+kafka_pid=$pid_dir/kafka/kafka.pid
+echo "Checking if kafka is running ..."
+
+if [ -f $kafka_pid ] && ps aux | grep -v grep | grep $(cat $kafka_pid) > /dev/null
+then
+	echo "Kafka is running as process `cat $kafka_pid`."
+else
+	echo "Error: Kafka is not running. Please start it via Ambari."
+	exit 1
+fi
+
+# Check storm if it has been started
+nimbus_pid=$pid_dir/storm/nimbus.pid
+supervisor_pid=$pid_dir/storm/supervisor.pid
+ui_pid=$pid_dir/storm/ui.pid
+echo "Checking if storm is running ..."
+
+if ! ([ -f $nimbus_pid ] && ps aux | grep -v grep | grep $(cat $nimbus_pid) >
/dev/null)
+then
+    echo "Error: Storm Nimbus is not running"
+    exit 1
+fi
+
+if ! ([ -f $supervisor_pid ] && ps aux | grep -v grep | grep $(cat $supervisor_pid)
> /dev/null)
+then
+    echo "Error: Storm Supervisor is not running"
+    exit 1
+fi
+
+if ! ([ -f $ui_pid ] && ps aux | grep -v grep | grep $(cat $ui_pid) > /dev/null)
+then
+    echo "Error: Storm UI is not running"
+    exit 1
+fi
+
+echo "Storm is running"
+
+
+###################################################################
+#              STEP 2: Starting Eagle Service
+###################################################################
+
+echo "STEP [2/3]: Start Eagle Service"
+$eagle_bin/eagle-service.sh start
+
+
+###################################################################
+#              STEP 3: Starting Eagle Topology
+###################################################################
+
+echo "STEP [3/3]: Init Eagle Service"
+$eagle_bin/eagle-service-init.sh
+[ $? != 0 ] && exit 1
+
+echo "Creating kafka topics for eagle ... "
+KAFKA_HOME=/usr/hdp/current/kafka-broker
+EAGLE_ZOOKEEPER_QUORUM=localhost:2181
+topic=`${KAFKA_HOME}/bin/kafka-topics.sh --list --zookeeper $EAGLE_ZOOKEEPER_QUORUM --topic
nn_jmx_metric_sandbox`
+if [ -z $topic ]; then
+	$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $EAGLE_ZOOKEEPER_QUORUM --replication-factor
1 --partitions 1 --topic nn_jmx_metric_sandbox
+fi
+
+if [ $? = 0 ]; then
+echo "==> Create kafka topic successfully for Hadoop Metric Monitoring"
+else
+echo "==> Failed, exiting"
+exit 1
+fi
+$eagle_bin/hadoop-metric-monitor.sh
+[ $? != 0 ] && exit 1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fba3aeba/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh b/eagle-hadoop-metric/src/main/resources/hadoop-metric-init.sh
old mode 100644
new mode 100755


Mime
View raw message