eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject incubator-eagle git commit: [EAGLE-798] Add a new alert publisher to store alerts into hbase
Date Fri, 25 Nov 2016 09:33:27 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 52dc7a5d9 -> 6a0529e96


[EAGLE-798] Add a new alert publisher to store alerts into hbase

https://issues.apache.org/jira/browse/EAGLE-798

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #683 from qingwen220/EAGLE-798.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6a0529e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6a0529e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6a0529e9

Branch: refs/heads/master
Commit: 6a0529e96e1f581c4fa476a82e5e897fc2509828
Parents: 52dc7a5
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Fri Nov 25 17:33:13 2016 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Fri Nov 25 17:33:13 2016 +0800

----------------------------------------------------------------------
 .../eagle-alert-parent/eagle-alert-app/pom.xml  |  5 ++
 .../eagle/alert/app/AlertEagleStorePlugin.java  | 93 ++++++++++++++++++++
 .../alert/engine/model/AlertPublishEvent.java   |  2 +
 .../eagle/alert/utils/AlertConstants.java       |  2 +
 .../publisher/impl/AlertEagleStorePlugin.java   |  2 +-
 .../eagle-metadata/eagle-metadata-base/pom.xml  |  5 ++
 .../eagle/metadata/model/AlertEntity.java       | 74 ++++++++++++++++
 .../metadata/model/AlertEntityRepository.java   | 27 ++++++
 .../app/dev/public/js/services/policySrv.js     |  5 --
 .../hdfs/HdfsTopologyEntityParser.java          | 14 +--
 10 files changed, 213 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
index d0760d5..400afaa 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
@@ -34,6 +34,11 @@
     </dependency>
       <dependency>
           <groupId>org.apache.eagle</groupId>
+          <artifactId>eagle-client-base</artifactId>
+          <version>${project.version}</version>
+      </dependency>
+      <dependency>
+          <groupId>org.apache.eagle</groupId>
           <artifactId>alert-coordinator</artifactId>
           <version>${project.version}</version>
       </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
new file mode 100644
index 0000000..a534012
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertEagleStorePlugin.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.alert.app;
+
+import com.google.common.base.Preconditions;
+import com.typesafe.config.Config;
+import org.apache.eagle.alert.engine.coordinator.Publishment;
+import org.apache.eagle.alert.engine.model.AlertStreamEvent;
+import org.apache.eagle.alert.engine.publisher.impl.AbstractPublishPlugin;
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.metadata.model.AlertEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.eagle.alert.engine.model.AlertPublishEvent.*;
+
+public class AlertEagleStorePlugin extends AbstractPublishPlugin {
+    private static final Logger LOG = LoggerFactory.getLogger(AlertEagleStorePlugin.class);
+    private IEagleServiceClient client;
+
+    @Override
+    public void init(Config config, Publishment publishment, Map conf) throws Exception {
+        super.init(config, publishment, conf);
+        client = new EagleServiceClientImpl(config.getString("service.host"), config.getInt("service.port"),
+                config.getString("service.username"), config.getString("service.password"));
+    }
+
+    @Override
+    public void close() {
+        try {
+            client.close();
+        } catch (IOException e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void onAlert(AlertStreamEvent event) throws Exception {
+        List<AlertStreamEvent> eventList = this.dedup(event);
+        if (eventList == null || eventList.isEmpty()) {
+            return;
+        }
+        List<AlertEntity> alertEvents = new ArrayList<>();
+        for (AlertStreamEvent e : eventList) {
+            alertEvents.add(convertAlertEvent(e));
+        }
+        client.create(alertEvents, AlertConstants.ALERT_SERVICE_ENDPOINT_NAME);
+    }
+
+    @Override
+    protected Logger getLogger() {
+        return LOG;
+    }
+
+    public AlertEntity convertAlertEvent(AlertStreamEvent event) {
+        Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before
being published: " + event.toString());
+        AlertEntity alertEvent = new AlertEntity();
+        Map<String, String> tags = new HashMap<>();
+        tags.put(POLICY_ID_KEY, event.getPolicyId());
+        tags.put(ALERT_ID_KEY, event.getAlertId());
+        if (event.getExtraData() != null && !event.getExtraData().isEmpty()) {
+            tags.put(SITE_ID_KEY, event.getExtraData().get(SITE_ID_KEY).toString());
+            alertEvent.setPolicyValue(event.getExtraData().get(POLICY_VALUE_KEY).toString());
+            alertEvent.setAppIds((List<String>) event.getExtraData().get(APP_IDS_KEY));
+        }
+        alertEvent.setTimestamp(event.getCreatedTime());
+        alertEvent.setAlertData(event.getDataMap());
+        alertEvent.setTags(tags);
+        return alertEvent;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
index 462e0fb..aee0ba0 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/model/AlertPublishEvent.java
@@ -33,8 +33,10 @@ public class AlertPublishEvent {
     private long alertTimestamp;
     private Map<String, Object> alertData;
 
+    public static final String ALERT_ID_KEY = "alertId";
     public static final String SITE_ID_KEY = "siteId";
     public static final String APP_IDS_KEY = "appIds";
+    public static final String POLICY_ID_KEY = "policyId";
     public static final String POLICY_VALUE_KEY = "policyValue";
 
     public String getAlertId() {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
index 2cbc977..566944f 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/utils/AlertConstants.java
@@ -24,4 +24,6 @@ public class AlertConstants {
 
     public static final String DEFAULT_SPOUT_NAME = "alertEngineSpout";
     public static final String DEFAULT_ROUTERBOLT_NAME = "streamRouterBolt";
+
+    public static final String ALERT_SERVICE_ENDPOINT_NAME = "AlertService";
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
index f31d850..48c3663 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEagleStorePlugin.java
@@ -50,7 +50,7 @@ public class AlertEagleStorePlugin extends AbstractPublishPlugin {
         try {
             client.close();
         } catch (IOException e) {
-            e.printStackTrace();
+            LOG.error(e.getMessage(), e);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml b/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
index b5a0b27..8df0674 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/pom.xml
@@ -60,5 +60,10 @@
             <artifactId>eagle-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-entity-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
new file mode 100644
index 0000000..c350dd9
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntity.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.metadata.model;
+
+import org.apache.eagle.alert.utils.AlertConstants;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Table("alert_detail")
+@ColumnFamily("f")
+@Prefix("alert_detail")
+@Service(AlertConstants.ALERT_SERVICE_ENDPOINT_NAME)
+@TimeSeries(true)
+@Tags({"alertId", "siteId", "policyId"})
+@Partition({"siteId"})
+@Indexes({
+        @Index(name = "Index_1_policyId", columns = { "policyId" }, unique = true)
+        })
+public class AlertEntity extends TaggedLogAPIEntity {
+    @Column("a")
+    private List<String> appIds;
+    @Column("b")
+    private String policyValue;
+    @Column("c")
+    private Map<String, Object> alertData;
+
+    public List<String> getAppIds() {
+        return appIds;
+    }
+
+    public void setAppIds(List<String> appIds) {
+        this.appIds = appIds;
+        valueChanged("appIds");
+    }
+
+    public String getPolicyValue() {
+        return policyValue;
+    }
+
+    public void setPolicyValue(String policyValue) {
+        this.policyValue = policyValue;
+        valueChanged("policyValue");
+    }
+
+    public Map<String, Object> getAlertData() {
+        return alertData;
+    }
+
+    public void setAlertData(Map<String, Object> alertData) {
+        this.alertData = alertData;
+        valueChanged("alertData");
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
new file mode 100644
index 0000000..c8219b6
--- /dev/null
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/AlertEntityRepository.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.metadata.model;
+
+import org.apache.eagle.log.entity.repo.EntityRepository;
+
+public class AlertEntityRepository extends EntityRepository {
+
+    public AlertEntityRepository() {
+        registerEntity(AlertEntity.class);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js b/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
index e3268f0..c2f3d8c 100644
--- a/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
+++ b/eagle-server/src/main/webapp/app/dev/public/js/services/policySrv.js
@@ -43,11 +43,6 @@
 					name: "Storage",
 					displayFields: [],
 					fields: []
-				},
-				'org.apache.eagle.alert.engine.publisher.impl.AlertFilePublisher': {
-					name: "LocalFile",
-					displayFields: ["fileName"],
-					fields: ["fileName", "rotate_every_kb", "number_of_files"]
 				}
 			},
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6a0529e9/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
index 72dce80..3cb55e9 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
@@ -100,7 +100,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
                 result.getMasterNodes().add(namenodeEntity);
                 numNamenode++;
                 if (namenodeEntity.getStatus().equalsIgnoreCase(NAME_NODE_ACTIVE_STATUS))
{
-                    createSlaveNodeEntities(url, timestamp, result);
+                    String namenodeVersion = createSlaveNodeEntities(url, timestamp, result);
+                    namenodeEntity.setVersion(namenodeVersion);
                 }
             } catch (RuntimeException ex) {
                 ex.printStackTrace();
@@ -118,16 +119,9 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
         final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
         final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME);
 
-        final String nameNodeUrlString = buildNamenodeInfo(url);
-        final Map<String, JMXBean> nameNodeMap = JMXQueryHelper.query(nameNodeUrlString);
-        final JMXBean nameNodeBean = nameNodeMap.get(JMX_NAMENODE_INFO);
-
         if (bean == null || bean.getPropertyMap() == null) {
             throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean
is null!");
         }
-        if (nameNodeBean == null || nameNodeBean.getPropertyMap() == null) {
-            throw new ServiceNotResponseException("Invalid JMX format, NameNode bean is null!");
-        }
 
         final String hostname = (String) bean.getPropertyMap().get(HA_NAME);
         HdfsServiceTopologyAPIEntity result = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE,
hostname, updateTime);
@@ -139,11 +133,10 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
         result.setUsedCapacityTB(Double.toString(capacityUsedGB / 1024));
         final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL);
         result.setNumBlocks(Integer.toString(blocksTotal));
-        result.setVersion((String) nameNodeBean.getPropertyMap().get(NAME_NODE_VERSION));
         return result;
     }
 
-    private void createSlaveNodeEntities(String url, long updateTime, TopologyEntityParserResult
result) throws IOException {
+    private String createSlaveNodeEntities(String url, long updateTime, TopologyEntityParserResult
result) throws IOException {
         final String urlString = buildNamenodeInfo(url);
         final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
         final JMXBean bean = jmxBeanMap.get(JMX_NAMENODE_INFO);
@@ -152,6 +145,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser
{
         }
         createAllDataNodeEntities(bean, updateTime, result);
         createAllJournalNodeEntities(bean, updateTime, result);
+        return (String) bean.getPropertyMap().get(NAME_NODE_VERSION);
     }
 
     private void createAllJournalNodeEntities(JMXBean bean, long updateTime, TopologyEntityParserResult
result) throws UnknownHostException {


Mime
View raw message