eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject incubator-eagle git commit: EAGLE-152 hadoop-metric monitoring automation script hadoop-metric-monitor.sh prepares everything for bringing up topology https://issues.apache.org/jira/browse/EAGLE-152 Author: yonzhang yonzhang2012@apache.org Closes: #85
Date Wed, 10 Feb 2016 21:18:32 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 0a9849398 -> caa2dfef5


EAGLE-152 hadoop-metric monitoring automation script
hadoop-metric-monitor.sh prepares everything for bringing up topology
https://issues.apache.org/jira/browse/EAGLE-152
Author: yonzhang yonzhang2012@apache.org
Closes: #85


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/caa2dfef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/caa2dfef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/caa2dfef

Branch: refs/heads/master
Commit: caa2dfef50edb6801edc06c4fc1ec4a25e3dccb8
Parents: 0a98493
Author: yonzhang <yonzhang@ebay.com>
Authored: Wed Feb 10 13:18:33 2016 -0800
Committer: yonzhang <yonzhang@ebay.com>
Committed: Wed Feb 10 13:18:33 2016 -0800

----------------------------------------------------------------------
 eagle-assembly/src/assembly/eagle-bin.xml       |  9 +++
 .../src/main/bin/hadoop-metric-monitor.sh       | 58 ++++++++++----------
 .../main/conf/sandbox-hadoopjmx-topology.conf   | 31 ++++++-----
 3 files changed, 55 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/caa2dfef/eagle-assembly/src/assembly/eagle-bin.xml
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/assembly/eagle-bin.xml b/eagle-assembly/src/assembly/eagle-bin.xml
index 05fd0d2..1c1a00f 100644
--- a/eagle-assembly/src/assembly/eagle-bin.xml
+++ b/eagle-assembly/src/assembly/eagle-bin.xml
@@ -233,5 +233,14 @@
                 <include>**</include>
             </includes>
         </fileSet>
+
+        <!-- copy hadoop metric configuration to assembly -->
+        <fileSet>
+            <directory>${project.basedir}/../eagle-hadoop-metric/src/main/resources</directory>
+            <outputDirectory>bin</outputDirectory>
+            <includes>
+                <include>hadoop-metric-init.sh</include>
+            </includes>
+        </fileSet>
     </fileSets>
 </assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/caa2dfef/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
index d88f9d9..7205ae3 100644
--- a/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
+++ b/eagle-assembly/src/main/bin/hadoop-metric-monitor.sh
@@ -15,34 +15,36 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 #
-#source $(dirname $0)/eagle-env.sh
-#
+source $(dirname $0)/eagle-env.sh
+
 ######################################################################
 ##            Import stream metadata for HADOOP METRIC
 ######################################################################
-### AlertDataSource: data sources bound to sites
-#echo "Importing AlertDataSourceService for HADOOP METRIC ... "
-#
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService"
-d '[{"prefix":"alertDataSource","tags":{"site" : "sandbox", "dataSource":"hadoop"}, "enabled":
"true", "config" : "", "desc":"HADOOP"}]'
-#
-#
-### AlertStreamService: alert streams generated from data source
-#echo ""
-#echo "Importing AlertStreamService for HADOOP METRIC ... "
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST \
-#-H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService"
\
-#-d '[{"prefix":"alertStream","tags":{"dataSource":"hadoop","streamName":"hadoopJmxMetric"},"desc":"hadoop
jmx metric stream"},{"prefix":"alertStream","tags":{"dataSource":"hadoop","streamName":"hadoopSysMetric"},"desc":"hadoop
system metric stream"}]'
-#
-### AlertExecutorService: what alert streams are consumed by alert executor
-#echo ""
-#echo "Importing AlertExecutorService for HADOOP METRIC ... "
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"
-d '[{"prefix":"alertExecutor","tags":{"dataSource":"hadoop","alertExecutorId":"hadoopJmxMetricExecutor","streamName":"hadoopJmxMetric"},"desc":"alert
executor for hadoop jmx stream"}]'
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService"
-d '[{"prefix":"alertExecutor","tags":{"dataSource":"hadoop","alertExecutorId":"hadoopSysMetricExecutor","streamName":"hadoopSysMetric"},"desc":"alert
executor for hadoop system stream"}]'
-#
-### AlertStreamSchemaService: schema for event from alert stream
-#echo ""
-#echo "Importing AlertStreamSchemaService for HADOOP METRIC ... "
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"
-d '[{"tags":{"dataSource":"hadoop","attrName":"host","streamName":"hadoopJmxMetric"},"attrType":"string","attrDescription":"metric
host","attrDisplayName":"host"},{"tags":{"dataSource":"hadoop","attrName":"site","streamName":"hadoopJmxMetric"},"attrType":"string","attrDescription":"metric
site","attrDisplayName":"site"},{"tags":{"dataSource":"hadoop","attrName":"timestamp","streamName":"hadoopJmxMetric"},"attrType":"long","attrDescription":"metric
timestamp","attrDisplayName":"timestamp"},{"tags":{"dataSource":"hadoop","attrName":"value","streamName":"hadoopJmxMetric"},"attrType":"string","attrDescription":"metric
value","attrDisplayName":"value"},{"tags":{"dataSource":"hadoop","attrName":"component","streamName":"hadoopJmxMetric"},"attrType":"string","attrDescription":
 "service component","attrDisplayName":"component"},{"tags":{"dataSource":"hadoop","attrName":"metric","streamName":"hadoopJmxMetric"},"attrType":"string","attrDescription":"metric
name","attrDisplayName":"metric"}]'
-#curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"
-d '[{"tags":{"dataSource":"hadoop","attrName":"host","streamName":"hadoopSysMetric"},"attrType":"string","attrDescription":"metric
host","attrDisplayName":"host"},{"tags":{"dataSource":"hadoop","attrName":"site","streamName":"hadoopSysMetric"},"attrType":"string","attrDescription":"metric
site","attrDisplayName":"site"},{"tags":{"dataSource":"hadoop","attrName":"timestamp","streamName":"hadoopSysMetric"},"attrType":"long","attrDescription":"metric
timestamp","attrDisplayName":"timestamp"},{"tags":{"dataSource":"hadoop","attrName":"value","streamName":"hadoopSysMetric"},"attrType":"string","attrDescription":"metric
value","attrDisplayName":"value"},{"tags":{"dataSource":"hadoop","attrName":"component","streamName":"hadoopSysMetric"},"attrType":"string","attrDescription":
 "service component","attrDisplayName":"component"},{"tags":{"dataSource":"hadoop","attrName":"metric","streamName":"hadoopSysMetric"},"attrType":"string","attrDescription":"metric
name","attrDisplayName":"metric"}]'
-#
-#$EAGLE_HOME/kafka-stream-monitor.sh hadoopJmxMetric hadoopJmxMetricExecutor $EAGLE_HOME/conf/sandbox-hadoopjmx-topology.conf
\ No newline at end of file
+${EAGLE_HOME}/bin/hadoop-metric-init.sh
+
+######################################################################
+##            Run topology for HADOOP METRIC
+######################################################################
+### if topology exists, we should shutdown it
+echo "check topology status ..."
+active=$(${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology status
| grep ACTIVE)
+echo "topology status $active"
+if [ "$active" ]; then
+ echo "stop topology ..."
+ ${EAGLE_HOME}/bin/eagle-topology.sh --topology sandbox-hadoopjmx-topology stop
+fi
+echo "start Eagle Hadoop Metric Monitoring topology"
+${EAGLE_HOME}/bin/eagle-topology.sh --main org.apache.eagle.hadoop.metric.HadoopJmxMetricMonitor
--topology sandbox-hadoopjmx-topology --config ../conf/sandbox-hadoopjmx-topology.conf start
+
+######################################################################
+##            Setup minutely crontab job for HADOOP METRIC
+######################################################################
+echo "set up crontab script"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cp $DIR/../tools/hadoop_jmx_collector/config-sample.json $DIR/../tools/hadoop_jmx_collector/config.json
+command="python $DIR/../tools/hadoop_jmx_collector/hadoop_jmx_kafka.py"
+echo $job
+job="* * * * * $command >> $DIR/../logs/hadoop_metric.log"
+cat <(fgrep -i -v "$command" <(crontab -l)) <(echo "$job") | crontab -
+
+echo "$(crontab -l)"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/caa2dfef/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf b/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf
index 4787de2..9e15fb8 100644
--- a/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf
+++ b/eagle-assembly/src/main/conf/sandbox-hadoopjmx-topology.conf
@@ -17,14 +17,12 @@
   "envContextConfig" : {
     "env" : "storm",
     "mode" : "cluster",
-    "topologyName" : "sandbox-hadoopjmx-topology",
-    "stormConfigFile" : "security-auditlog-storm.yaml",
+    "topologyName" : "hadoopJmxMetricTopology",
+    "stormConfigFile" : "hadoopjmx.yaml",
     "parallelismConfig" : {
       "kafkaMsgConsumer" : 1,
-      "hadoopJmxMetricExecutor*" : 1
-    },
-    "nimbusHost":"sandbox.hortonworks.com",
-    "nimbusThriftPort":6627
+      "hadoopJmxMetricAlertExecutor*" : 1
+    }
   },
   "dataSourceConfig": {
     "topic" : "nn_jmx_metric_sandbox",
@@ -32,33 +30,36 @@
     "zkConnectionTimeoutMS" : 15000,
     "consumerGroupId" : "EagleConsumer",
     "fetchSize" : 1048586,
-    "deserializerClass" : "org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer",
+    "deserializerClass" : "org.apache.eagle.datastream.storm.JsonMessageDeserializer",
     "transactionZKServers" : "sandbox.hortonworks.com",
     "transactionZKPort" : 2181,
     "transactionZKRoot" : "/consumers",
-    "consumerGroupId" : "eagle.hadoop.consumer",
     "transactionStateUpdateMS" : 2000
   },
   "alertExecutorConfigs" : {
-     "hadoopJmxMetricExecutor" : {
-       "parallelism" : 1,
-       "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
-       "needValidation" : "true"
-     }
+    "hadoopJmxMetricAlertExecutor" : {
+      "parallelism" : 1,
+      "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
+      "needValidation" : "true"
+    }
   },
   "eagleProps" : {
     "site" : "sandbox",
-    "dataSource": "hadoop",
+    "dataSource": "hadoopJmxMetricDataSource",
     "dataJoinPollIntervalSec" : 30,
     "mailHost" : "mailHost.com",
     "mailSmtpPort":"25",
     "mailDebug" : "true",
+    "balancePartitionEnabled" : true,
+    #"partitionRefreshIntervalInMin" : 60,
+    #"kafkaStatisticRangeInMin" : 60,
     "eagleService": {
       "host": "localhost",
-      "port": 9099
+      "port": 9099,
       "username": "admin",
       "password": "secret"
     }
+    "readHdfsUserCommandPatternFrom" : "file"
   },
   "dynamicConfigSource" : {
     "enabled" : true,


Mime
View raw message