eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: [EAGLE-650] Hadoop JMX Metric Monitoring Application
Date Mon, 24 Oct 2016 11:38:12 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 3ff3e55ac -> 698b30282


[EAGLE-650] Hadoop JMX Metric Monitoring Application

Hadoop JMX Metric Monitoring Application

https://issues.apache.org/jira/browse/EAGLE-650

Monitored Services:
* hdfs name node
* yarn resource manager node
* hbase master node

Author: Hao Chen <hao@apache.org>

Closes #552 from haoch/EAGLE-650.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/698b3028
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/698b3028
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/698b3028

Branch: refs/heads/master
Commit: 698b302821c7a89cee1172dbf8d2cdb296506b54
Parents: 3ff3e55
Author: Hao Chen <hao@apache.org>
Authored: Mon Oct 24 19:37:58 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Mon Oct 24 19:37:58 2016 +0800

----------------------------------------------------------------------
 ...e.alert.app.AlertUnitTopologyAppProvider.xml |  3 +-
 .../app/environment/impl/StaticEnvironment.java | 19 +---
 .../eagle/app/service/ApplicationAction.java    | 14 +--
 .../apache/eagle/app/sink/KafkaStreamSink.java  | 39 ++++++--
 .../app/environment/StaticEnvironmentTest.java  | 10 ++-
 .../apache/eagle/metadata/model/StreamDesc.java |  3 +-
 .../metadata/utils/StreamIdConversions.java     | 46 ++++++++++
 .../metadata/utils/StreamIdConversionsTest.java | 37 ++++++++
 ...e.app.example.ExampleApplicationProvider.xml |  3 +-
 ...apache.eagle.gc.GCLogApplicationProvider.xml |  5 +-
 eagle-hadoop-metric/pom.xml                     |  6 +-
 .../metric/HadoopMetricMonitorAppProdiver.java  | 23 +++++
 ...le.metric.HadoopMetricMonitorAppProdiver.xml | 94 ++++++++++++++++++++
 ...org.apache.eagle.app.spi.ApplicationProvider | 16 ++++
 .../HadoopMetricMonitorAppProdiverTest.java     | 84 +++++++++++++++++
 ...doop.queue.HadoopQueueRunningAppProvider.xml |  2 +-
 ...gregation.AggregationApplicationProvider.xml |  2 +-
 ....security.hbase.HBaseAuditLogAppProvider.xml |  4 +-
 ...ecurity.auditlog.HdfsAuditLogAppProvider.xml |  2 +-
 eagle-server/pom.xml                            |  7 ++
 20 files changed, 370 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
index 28f7db4..7b12917 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
@@ -18,7 +18,8 @@
 
 <application>
     <type>AlertUnitTopologyApp</type>
-    <name>Alert Unit Topology Application</name>
+    <name>Alert Engine</name>
+    <description>Real-time Alert Engine</description>
     <version>0.5.0-incubating</version>
     <configuration>
        <!-- alert topology sizing parameters -->

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
index 1c297fb..1c17b1f 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StaticEnvironment.java
@@ -17,23 +17,10 @@
 package org.apache.eagle.app.environment.impl;
 
 import com.typesafe.config.Config;
-import org.apache.eagle.app.environment.Environment;
-import org.apache.eagle.app.sink.StreamSinkProvider;
-
-public class StaticEnvironment implements Environment {
-    private final Config config;
+import org.apache.eagle.app.environment.AbstractEnvironment;
 
+public class StaticEnvironment extends AbstractEnvironment {
     public StaticEnvironment(Config config) {
-        this.config = config;
-    }
-
-    @Override
-    public Config config() {
-        return this.config;
-    }
-
-    @Override
-    public StreamSinkProvider streamSink() {
-        throw new IllegalStateException("streamSink() is not supported in " + StaticEnvironment.class.getSimpleName());
+        super(config);
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
index 557737f..b7258d3 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationAction.java
@@ -31,6 +31,7 @@ import org.apache.eagle.app.environment.ExecutionRuntime;
 import org.apache.eagle.app.environment.ExecutionRuntimeManager;
 import org.apache.eagle.app.sink.KafkaStreamSinkConfig;
 import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
 import org.apache.eagle.metadata.model.StreamDesc;
 import org.apache.eagle.metadata.model.StreamSinkConfig;
 import org.slf4j.Logger;
@@ -86,21 +87,14 @@ public class ApplicationAction implements Serializable {
         this.alertMetadataService = alertMetadataService;
     }
 
-    /**
-     * Generate global unique streamId to install.
-     * TODO refactor with streamId and siteId
-     */
-    private static String generateUniqueStreamId(String siteId, String streamTypeId) {
-        return String.format("%s_%s", streamTypeId, siteId).toUpperCase();
-    }
-
     public void doInstall() {
         if (metadata.getDescriptor().getStreams() != null) {
             List<StreamDesc> streamDescToInstall = metadata.getDescriptor().getStreams().stream().map((streamDefinition
-> {
                 StreamDefinition copied = streamDefinition.copy();
                 copied.setSiteId(metadata.getSite().getSiteId());
-                copied.setStreamId(generateUniqueStreamId(metadata.getSite().getSiteId(),
copied.getStreamId()));
-                StreamSinkConfig streamSinkConfig = this.runtime.environment().streamSink().getSinkConfig(copied.getStreamId(),
this.effectiveConfig);
+                copied.setStreamId(StreamIdConversions.formatSiteStreamId(metadata.getSite().getSiteId(),
copied.getStreamId()));
+                StreamSinkConfig streamSinkConfig = this.runtime.environment()
+                    .streamSink().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(),
copied.getStreamId()), this.effectiveConfig);
                 StreamDesc streamDesc = new StreamDesc();
                 streamDesc.setSchema(copied);
                 streamDesc.setSink(streamSinkConfig);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
index cf5351b..d6d80f4 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/sink/KafkaStreamSink.java
@@ -24,6 +24,7 @@ import com.typesafe.config.Config;
 import kafka.javaapi.producer.Producer;
 import kafka.producer.KeyedMessage;
 import kafka.producer.ProducerConfig;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -96,18 +97,42 @@ public class KafkaStreamSink extends StormStreamSink<KafkaStreamSinkConfig>
{
     }
 
     public static class Provider implements StreamSinkProvider<KafkaStreamSink, KafkaStreamSinkConfig>
{
+        private static final Logger LOG = LoggerFactory.getLogger(Provider.class);
+        private static final String DEAULT_SHARED_TOPIC_CONF_KEY = "dataSinkConfig.topic";
+
+        private String getStreamSpecificTopicConfigKey(String streamId) {
+            return String.format("dataSinkConfig.%s.topic",streamId);
+        }
+
         @Override
         public KafkaStreamSinkConfig getSinkConfig(String streamId, Config config) {
             KafkaStreamSinkConfig desc = new KafkaStreamSinkConfig();
-            desc.setTopicId(config.getString("dataSinkConfig.topic"));
+            String streamSpecificTopicConfigKey = getStreamSpecificTopicConfigKey(streamId);
+            if (config.hasPath(streamSpecificTopicConfigKey)) {
+                desc.setTopicId(streamSpecificTopicConfigKey);
+            } else if (config.hasPath(DEAULT_SHARED_TOPIC_CONF_KEY)) {
+                desc.setTopicId(config.getString(DEAULT_SHARED_TOPIC_CONF_KEY));
+                LOG.warn("Using default shared topic {}: {}", DEAULT_SHARED_TOPIC_CONF_KEY,
desc.getTopicId());
+            } else {
+                LOG.error("Neither stream specific topic: {} nor default shared topic: {}
found in config", streamSpecificTopicConfigKey, DEAULT_SHARED_TOPIC_CONF_KEY);
+                throw new IllegalArgumentException("Neither stream specific topic: "
+                    + streamSpecificTopicConfigKey + " nor default shared topic: " + DEAULT_SHARED_TOPIC_CONF_KEY
+ " found in config");
+            }
             desc.setBrokerList(config.getString("dataSinkConfig.brokerList"));
-            desc.setSerializerClass(config.getString("dataSinkConfig.serializerClass"));
-            desc.setKeySerializerClass(config.getString("dataSinkConfig.keySerializerClass"));
+            desc.setSerializerClass(config.hasPath("dataSinkConfig.serializerClass")
+                ? config.getString("dataSinkConfig.serializerClass") : "kafka.serializer.StringEncoder");
+            desc.setKeySerializerClass(config.hasPath("dataSinkConfig.keySerializerClass")
+                ? config.getString("dataSinkConfig.keySerializerClass") : "kafka.serializer.StringEncoder");
+
             // new added properties for async producer
-            desc.setNumBatchMessages(config.getString("dataSinkConfig.numBatchMessages"));
-            desc.setProducerType(config.getString("dataSinkConfig.producerType"));
-            desc.setMaxQueueBufferMs(config.getString("dataSinkConfig.maxQueueBufferMs"));
-            desc.setRequestRequiredAcks(config.getString("dataSinkConfig.requestRequiredAcks"));
+            desc.setNumBatchMessages(config.hasPath("dataSinkConfig.numBatchMessages")
+                ? config.getString("dataSinkConfig.numBatchMessages") : "1024");
+            desc.setProducerType(config.hasPath("dataSinkConfig.producerType")
+                ? config.getString("dataSinkConfig.producerType") : "async");
+            desc.setMaxQueueBufferMs(config.hasPath("dataSinkConfig.maxQueueBufferMs")
+                ? config.getString("dataSinkConfig.maxQueueBufferMs") : "3000");
+            desc.setRequestRequiredAcks(config.hasPath("dataSinkConfig.requestRequiredAcks")
+                ? config.getString("dataSinkConfig.requestRequiredAcks") : "1");
             return desc;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
index 797eba8..de64e6a 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/StaticEnvironmentTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.eagle.app.environment;
 
+import com.typesafe.config.ConfigFactory;
 import org.apache.eagle.app.environment.impl.StaticEnvironment;
 import org.junit.Before;
 import org.junit.Test;
@@ -25,11 +26,14 @@ public class StaticEnvironmentTest {
 
     @Before
     public void setUp(){
-        environment = new StaticEnvironment(null);
+        environment = new StaticEnvironment(ConfigFactory.load());
     }
 
-    @Test(expected = IllegalStateException.class)
-    public void testNotSupport(){
+    /**
+     * StaticEnvironment stream sink will be usefully for collection only app
+     */
+    @Test
+    public void testStreamSinkSupport(){
         environment.streamSink();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
index abf6a81..47d16fd 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/StreamDesc.java
@@ -17,6 +17,7 @@
 package org.apache.eagle.metadata.model;
 
 import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
+import org.apache.eagle.metadata.utils.StreamIdConversions;
 
 public class StreamDesc {
     private String streamId;
@@ -28,7 +29,7 @@ public class StreamDesc {
     }
 
     public void setStreamId(String streamId) {
-        this.streamId = streamId;
+        this.streamId = StreamIdConversions.formStreamTypeId(streamId);
     }
 
     public StreamDefinition getSchema() {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
new file mode 100644
index 0000000..b5f5631
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/utils/StreamIdConversions.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metadata.utils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * The name conversion about streamId:
+ * 1. Case insensitive.
+ * 2. All in upper case.
+ */
+public class StreamIdConversions {
+
+    public static String formStreamTypeId(String streamTypeId) {
+        return streamTypeId.toUpperCase();
+    }
+
+    public static String formatSiteStreamId(String siteId, String streamTypeId) {
+        return String.format("%s_%s", streamTypeId, siteId).toUpperCase();
+    }
+
+    public static String parseStreamTypeId(String siteId, String generatedUniqueStreamId)
{
+        String subffix = String.format("_%s", siteId).toUpperCase();
+        if (generatedUniqueStreamId.endsWith(subffix)) {
+            int streamTypeIdLength = generatedUniqueStreamId.length() - subffix.length();
+            Preconditions.checkArgument(streamTypeIdLength > 0, "Invalid streamId: " +
generatedUniqueStreamId + ", streamTypeId is empty");
+            return generatedUniqueStreamId.substring(0, streamTypeIdLength).toUpperCase();
+        } else {
+            throw new IllegalArgumentException("Invalid streamId: " + generatedUniqueStreamId
+ ", not end with \"" + subffix + "\"");
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
new file mode 100644
index 0000000..f0018bf
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/test/java/org/apache/eagle/metadata/utils/StreamIdConversionsTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metadata.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StreamIdConversionsTest {
+    @Test
+    public void testGenerateUniqueStreamId() {
+        Assert.assertEquals("MOCK_STREAM_TEST", StreamIdConversions.formatSiteStreamId("test","mock_stream"));
+    }
+
+    @Test
+    public void testFormatStreamId() {
+        Assert.assertEquals("MOCK_STREAM", StreamIdConversions.formStreamTypeId("mock_stream"));
+    }
+
+    @Test
+    public void testParseStreamTypeId() {
+        Assert.assertEquals("MOCK_STREAM", StreamIdConversions.parseStreamTypeId("test","MOCK_STREAM_TEST"));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
b/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
index 432b92f..c1e8711 100644
--- a/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
+++ b/eagle-examples/eagle-app-example/src/main/resources/META-INF/providers/org.apache.eagle.app.example.ExampleApplicationProvider.xml
@@ -18,9 +18,8 @@
 
 <application>
     <type>EXAMPLE_APPLICATION</type>
-    <name>Example Monitoring Application</name>
+    <name>Example Monitor APP</name>
     <version>0.5.0-incubating</version>
-    <appClass>org.apache.eagle.app.example.ExampleStormApplication</appClass>
     <viewPath>/apps/example</viewPath>
     <configuration>
         <property>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
index a0477a2..c5f9732 100644
--- a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
+++ b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml
@@ -21,10 +21,9 @@
   -->
 
 <application>
-    <type>GCLogApplication</type>
-    <name>GC Log Monitoring Application</name>
+    <type>GC_LOG_MONITOR</type>
+    <name>GC Log Monitor</name>
     <version>0.5.0-incubating</version>
-    <appClass>org.apache.eagle.gc.GCLogApplication</appClass>
     <configuration>
         <!-- topology level configurations -->
         <property>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/pom.xml b/eagle-hadoop-metric/pom.xml
index b397e37..33a7b90 100644
--- a/eagle-hadoop-metric/pom.xml
+++ b/eagle-hadoop-metric/pom.xml
@@ -26,6 +26,10 @@
 
     <artifactId>eagle-hadoop-metric</artifactId>
     <dependencies>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-app-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
-
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
b/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
new file mode 100644
index 0000000..e6ebde1
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiver.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metric;
+
+import org.apache.eagle.app.StaticApplicationProvider;
+
+public class HadoopMetricMonitorAppProdiver extends StaticApplicationProvider {
+    // Metadata: META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
new file mode 100644
index 0000000..752c0cb
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/META-INF/providers/org.apache.eagle.metric.HadoopMetricMonitorAppProdiver.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<application>
+    <type>HADOOP_JMX_METRIC_MONITOR</type>
+    <name>Hadoop JMX Metric Monitor</name>
+    <version>0.5.0-incubating</version>
+    <configuration>
+        <!-- data source configurations -->
+        <property>
+            <name>dataSinkConfig.HADOOP_JMX_METRIC_STREAM.topic</name>
+            <displayName>JMX Metric Kafka Topic</displayName>
+            <value>hadoop_jmx_metric_{SITE_ID}</value>
+            <description>Hadoop JMX metric kafka topic name for stream: HADOOP_JMX_METRIC_STREAM</description>
+            <required>true</required>
+        </property>
+        <property>
+            <name>dataSinkConfig.brokerList</name>
+            <displayName>Kafka Brokers</displayName>
+            <value>localhost:6667</value>
+            <description>Sourced Kafka Brokers like broker1,broker2,...</description>
+            <required>true</required>
+        </property>
+    </configuration>
+    <streams>
+        <stream>
+            <streamId>HADOOP_JMX_METRIC_STREAM</streamId>
+            <description>Hadoop JMX Metric Stream including name node, resource manager,
etc.</description>
+            <validate>true</validate>
+            <timeseries>true</timeseries>
+            <columns>
+                <column>
+                    <name>host</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>timestamp</name>
+                    <type>long</type>
+                </column>
+                <column>
+                    <name>metric</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>component</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>site</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>value</name>
+                    <type>double</type>
+                    <defaultValue>0.0</defaultValue>
+                </column>
+            </columns>
+        </stream>
+    </streams>
+    <docs>
+        <install>
+            <b>How to Install</b>
+            <ol>
+                <li>Create a kafka topic named <code>hadoop_jmx_metric_{SITE_ID}</code></li>
+                <li>Setup metric collector for monitored Hadoop/HBase using <a target="_blank"
href="https://github.com/apache/incubator-eagle/tree/master/eagle-external/hadoop_jmx_collector">hadoop_jmx_collector</a></li>
+                <li>Setup <code>Kafka Topic Name</code> and <code>Kafka
Broker List </code> in <code>Settings</code> tab</li>
+                <li>Click "Install" button then you will see the <code>HADOOP_JMX_METRIC_STREAM_{SITE_ID}</code>
in <a href="/#/alert/streamList">Streams</a></li>
+            </ol>
+        </install>
+        <uninstall>
+            <b>How to Uninstall</b>
+            <ol>
+                <li>Stop and remove installed <code>hadoop_jmx_collector</code></li>
+                <li>Remove kafka topic <code>hadoop_jmx_metric_{SITE_ID}</code></li>
+                <li>Click "Uninstall" button which will remove stream <code>HADOOP_JMX_METRIC_STREAM_{SITE_ID}</code></li>
+            </ol>
+        </uninstall>
+    </docs>
+</application>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
b/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
new file mode 100644
index 0000000..25d911d
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.eagle.metric.HadoopMetricMonitorAppProdiver
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
----------------------------------------------------------------------
diff --git a/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
b/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
new file mode 100644
index 0000000..ee0b3c0
--- /dev/null
+++ b/eagle-hadoop-metric/src/main/test/java/org/apache/eagle/metric/HadoopMetricMonitorAppProdiverTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.metric;
+
+
+import com.google.inject.Inject;
+import org.apache.eagle.app.resource.ApplicationResource;
+import org.apache.eagle.app.service.ApplicationOperations;
+import org.apache.eagle.app.test.ApplicationSimulator;
+import org.apache.eagle.app.test.ApplicationTestBase;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.model.SiteEntity;
+import org.apache.eagle.metadata.resource.SiteResource;
+import org.apache.eagle.metadata.service.ApplicationStatusUpdateService;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class HadoopMetricMonitorAppProdiverTest extends ApplicationTestBase {
+
+    @Inject
+    private SiteResource siteResource;
+    @Inject
+    private ApplicationResource applicationResource;
+    @Inject
+    private ApplicationSimulator simulator;
+    @Inject
+    ApplicationStatusUpdateService statusUpdateService;
+
+    @Test
+    public void testApplicationLifecycle() throws InterruptedException {
+        // Create local site
+        SiteEntity siteEntity = new SiteEntity();
+        siteEntity.setSiteId("test_site");
+        siteEntity.setSiteName("Test Site");
+        siteEntity.setDescription("Test Site for HADOOP_JMX_METRIC_MONITOR");
+        siteResource.createSite(siteEntity);
+        Assert.assertNotNull(siteEntity.getUuid());
+
+        ApplicationOperations.InstallOperation installOperation = new ApplicationOperations.InstallOperation("test_site",
"HADOOP_JMX_METRIC_MONITOR", ApplicationEntity.Mode.LOCAL);
+        installOperation.setConfiguration(getConf());
+        // Install application
+        ApplicationEntity applicationEntity = applicationResource.installApplication(installOperation).getData();
+        // Uninstall application
+        applicationResource.uninstallApplication(new ApplicationOperations.UninstallOperation(applicationEntity.getUuid()));
+        try {
+            applicationResource.getApplicationEntityByUUID(applicationEntity.getUuid());
+            Assert.fail("Application instance (UUID: " + applicationEntity.getUuid() + ")
should have been uninstalled");
+        } catch (Exception ex) {
+            // Expected exception
+        }
+    }
+
+    private Map<String, Object> getConf() {
+        Map<String, Object> conf = new HashMap<>();
+        conf.put("dataSinkConfig.topic", "testTopic");
+        conf.put("dataSinkConfig.brokerList", "broker");
+        conf.put("dataSinkConfig.serializerClass", "serializerClass");
+        conf.put("dataSinkConfig.keySerializerClass", "keySerializerClass");
+        conf.put("dataSinkConfig.producerType", "async");
+        conf.put("dataSinkConfig.numBatchMessages", 4096);
+        conf.put("dataSinkConfig.maxQueueBufferMs", 5000);
+        conf.put("dataSinkConfig.requestRequiredAcks", 0);
+        conf.put("spoutNum", 2);
+        conf.put("mode", "LOCAL");
+        return conf;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index 02b60ef..c5e9ff9 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -18,7 +18,7 @@
 
 <application>
   <type>HADOOP_QUEUE_RUNNING_APP</type>
-  <name>Hadoop Queue Running Monitoring</name>
+  <name>Hadoop Queue Monitor</name>
   <version>0.5.0-incubating</version>
   <configuration>
     <!-- org.apache.eagle.hadoop.queue.HadoopQueueRunningAppConfig -->

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
index 29ca98a..2c8d60c 100644
--- a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
@@ -18,7 +18,7 @@
 
 <application>
     <type>MR_JPM_AGGREGATION_APP</type>
-    <name>Map Reduce Job Metrics Aggregation</name>
+    <name>MR Metrics Aggregation</name>
     <version>0.5.0-incubating</version>
     <configuration>
         <property>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
index 8f648a1..1cfd068 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
@@ -17,8 +17,8 @@
   -->
 
 <application>
-    <type>HBaseAuditLogApplication</type>
-    <name>Hbase Audit Log Monitoring Application</name>
+    <type>HBASE_AUDIT_LOG_MONITOR</type>
+    <name>Hbase Audit Log Monitor</name>
     <version>0.5.0-incubating</version>
     <appClass>org.apache.eagle.security.hbase.HBaseAuditLogApplication</appClass>
     <configuration>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index 2c02a8f..bb302c1 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -22,7 +22,7 @@
 
 <application>
     <type>HdfsAuditLogApplication</type>
-    <name>Hdfs Audit Log Monitoring Application</name>
+    <name>Hdfs Audit Log Monitor</name>
     <version>0.5.0-incubating</version>
     <configuration>
         <!-- topology related configurations -->

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/698b3028/eagle-server/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml
index 1a3e7c9..f58a166 100644
--- a/eagle-server/pom.xml
+++ b/eagle-server/pom.xml
@@ -328,6 +328,13 @@
                     <artifactId>eagle-topology-app</artifactId>
                     <version>${project.version}</version>
                 </dependency>
+
+                <!--- App: HadoopMetricMonitorApp -->
+                <dependency>
+                    <groupId>org.apache.eagle</groupId>
+                    <artifactId>eagle-hadoop-metric</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
             </dependencies>
         </profile>
     </profiles>



Mime
View raw message