metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject metron git commit: METRON-1255 MetaAlert search is not filtering on status (merrimanr) closes apache/metron#802
Date Wed, 18 Oct 2017 21:21:51 GMT
Repository: metron
Updated Branches:
  refs/heads/master fef8833c1 -> 73623ec74


METRON-1255 MetaAlert search is not filtering on status (merrimanr) closes apache/metron#802


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73623ec7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73623ec7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73623ec7

Branch: refs/heads/master
Commit: 73623ec74232765849f40480add9117da67c11d5
Parents: fef8833
Author: merrimanr <merrimanr@gmail.com>
Authored: Wed Oct 18 16:21:37 2017 -0500
Committer: merrimanr <merrimanr@apache.org>
Committed: Wed Oct 18 16:21:37 2017 -0500

----------------------------------------------------------------------
 .../dao/ElasticsearchMetaAlertDao.java          |  13 +-
 .../ElasticsearchMetaAlertIntegrationTest.java  | 163 ++++++++++++++++++-
 .../metron/indexing/dao/MetaAlertDao.java       |   2 +-
 3 files changed, 168 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index a64f5ad..44c983e 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -21,6 +21,7 @@ package org.apache.metron.elasticsearch.dao;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 
@@ -177,15 +178,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
   public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException
{
     // Wrap the query to also get any meta-alerts.
     QueryBuilder qb = constantScoreQuery(boolQuery()
-        .should(new QueryStringQueryBuilder(searchRequest.getQuery()))
-        .should(boolQuery()
-            .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()))
-            .must(nestedQuery(
+        .must(boolQuery()
+            .should(new QueryStringQueryBuilder(searchRequest.getQuery()))
+            .should(nestedQuery(
                 ALERT_FIELD,
                 new QueryStringQueryBuilder(searchRequest.getQuery())
                 )
             )
         )
+        // Ensures that it's a meta alert with active status or that it's an alert (signified
by having no status field)
+        .must(boolQuery()
+            .should(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()))
+            .should(boolQuery().mustNot(existsQuery(MetaAlertDao.STATUS_FIELD)))
+        )
     );
     return elasticsearchDao.search(searchRequest, qb);
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index c562e0b..12ac888 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -21,16 +21,20 @@ package org.apache.metron.elasticsearch.integration;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import java.io.File;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
@@ -40,6 +44,9 @@ import org.apache.metron.elasticsearch.integration.components.ElasticSearchCompo
 import org.apache.metron.indexing.dao.AccessConfig;
 import org.apache.metron.indexing.dao.IndexDao;
 import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SortField;
 import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.indexing.dao.update.ReplaceRequest;
 import org.junit.AfterClass;
@@ -83,6 +90,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
         put("es.date.format", DATE_FORMAT);
       }
     };
+    accessConfig.setMaxSearchResults(1000);
     accessConfig.setGlobalConfigSupplier(() -> globalConfig);
 
     esDao = new ElasticsearchDao();
@@ -198,7 +206,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
 
       {
         //ensure alerts in ES are up-to-date
-        boolean found = findUpdatedDoc(message0, guid);
+        boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME);
         Assert.assertTrue("Unable to find updated document", found);
         long cnt = 0;
         for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS))
{
@@ -257,7 +265,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
         }
       }, Optional.empty());
 
-      boolean found = findUpdatedDoc(message0, guid);
+      boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME);
       Assert.assertTrue("Unable to find updated document", found);
       {
         //ensure ES is up-to-date
@@ -302,12 +310,157 @@ public class ElasticsearchMetaAlertIntegrationTest {
     }
   }
 
-  protected boolean findUpdatedDoc(Map<String, Object> message0, String guid)
+  /**
+   {
+     "guid": "active_metaalert",
+     "source:type": "metaalert",
+     "alert": [],
+     "status": "active"
+   }
+   */
+  @Multiline
+  public static String activeMetaAlert;
+
+  /**
+   {
+     "guid": "inactive_metaalert",
+     "source:type": "metaalert",
+     "alert": [],
+     "status": "inactive"
+   }
+   */
+  @Multiline
+  public static String inactiveMetaAlert;
+
+  @Test
+  public void shouldSearchByStatus() throws Exception {
+    List<Map<String, Object>> metaInputData = new ArrayList<>();
+    Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert,
new TypeReference<Map<String, Object>>() {});
+    metaInputData.add(activeMetaAlertJSON);
+    Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert,
new TypeReference<Map<String, Object>>() {});
+    metaInputData.add(inactiveMetaAlertJSON);
+
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
+    // Wait for updates to persist
+    findUpdatedDoc(inactiveMetaAlertJSON, "inactive_metaalert", MetaAlertDao.METAALERT_TYPE);
+
+    SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+      {
+        setQuery("*");
+        setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+        setFrom(0);
+        setSize(5);
+        setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+      }
+    });
+    Assert.assertEquals(1, searchResponse.getTotal());
+    Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), searchResponse.getResults().get(0).getSource().get(MetaAlertDao.STATUS_FIELD));
+  }
+
+  /**
+   {
+   "guid": "search_by_nested_alert_active_0",
+   "source:type": "test",
+   "ip_src_addr": "192.168.1.1",
+   "ip_src_port": 8010
+   }
+   */
+  @Multiline
+  public static String searchByNestedAlertActive0;
+
+  /**
+   {
+   "guid": "search_by_nested_alert_inactive_1",
+   "source:type": "test",
+   "ip_src_addr": "192.168.1.2",
+   "ip_src_port": 8009
+   }
+   */
+  @Multiline
+  public static String searchByNestedAlertActive1;
+
+  /**
+   {
+   "guid": "search_by_nested_alert_inactive_0",
+   "source:type": "test",
+   "ip_src_addr": "192.168.1.3",
+   "ip_src_port": 8008
+   }
+   */
+  @Multiline
+  public static String searchByNestedAlertInactive0;
+
+  /**
+   {
+   "guid": "search_by_nested_alert_inactive_1",
+   "source:type": "test",
+   "ip_src_addr": "192.168.1.4",
+   "ip_src_port": 8007
+   }
+   */
+  @Multiline
+  public static String searchByNestedAlertInactive1;
+
+  @Test
+  public void shouldSearchByNestedAlert() throws Exception {
+    // Create alerts
+    List<Map<String, Object>> alerts = new ArrayList<>();
+    Map<String, Object> searchByNestedAlertActive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive0,
new TypeReference<Map<String, Object>>() {});
+    alerts.add(searchByNestedAlertActive0JSON);
+    Map<String, Object> searchByNestedAlertActive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive1,
new TypeReference<Map<String, Object>>() {});
+    alerts.add(searchByNestedAlertActive1JSON);
+    Map<String, Object> searchByNestedAlertInactive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive0,
new TypeReference<Map<String, Object>>() {});
+    alerts.add(searchByNestedAlertInactive0JSON);
+    Map<String, Object> searchByNestedAlertInactive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive1,
new TypeReference<Map<String, Object>>() {});
+    alerts.add(searchByNestedAlertInactive1JSON);
+    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
+    // Wait for updates to persist
+    findUpdatedDoc(searchByNestedAlertInactive1JSON, "search_by_nested_alert_inactive_1",
SENSOR_NAME);
+
+    // Create metaalerts
+    Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert,
new TypeReference<Map<String, Object>>() {});
+    activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertActive0JSON, searchByNestedAlertActive1JSON));
+    Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert,
new TypeReference<Map<String, Object>>() {});
+    inactiveMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertInactive0JSON, searchByNestedAlertInactive1JSON));
+
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    elasticsearchAdd(Arrays.asList(activeMetaAlertJSON, inactiveMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX,
MetaAlertDao.METAALERT_TYPE);
+    // Wait for updates to persist
+    findUpdatedDoc(activeMetaAlertJSON, "active_metaalert", MetaAlertDao.METAALERT_TYPE);
+
+    SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+      {
+        setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1
AND alert.ip_src_port:8009)");
+        setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+        setFrom(0);
+        setSize(5);
+        setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+      }
+    });
+    // Should not have results because nested alerts shouldn't be flattened
+    Assert.assertEquals(0, searchResponse.getTotal());
+
+    searchResponse = metaDao.search(new SearchRequest() {
+      {
+        setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010) OR (alert.ip_src_addr:192.168.1.1
AND alert.ip_src_port:8010)");
+        setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+        setFrom(0);
+        setSize(5);
+        setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+      }
+    });
+    // Nested query should match a nested alert
+    Assert.assertEquals(1, searchResponse.getTotal());
+    Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid"));
+  }
+
+  protected boolean findUpdatedDoc(Map<String, Object> message0, String guid, String
sensorType)
       throws InterruptedException, IOException {
     boolean found = false;
     for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) {
-      Document doc = metaDao.getLatest(guid, SENSOR_NAME);
-      if (message0.equals(doc.getDocument())) {
+      Document doc = metaDao.getLatest(guid, sensorType);
+      if (doc != null && message0.equals(doc.getDocument())) {
         found = true;
       }
     }

http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
index 05746c4..e9f047b 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
@@ -27,7 +27,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
 
 public interface MetaAlertDao extends IndexDao {
 
-  String METAALERTS_INDEX = "metaalert";
+  String METAALERTS_INDEX = "metaalert_index";
   String METAALERT_TYPE = "metaalert";
   String METAALERT_DOC = METAALERT_TYPE + "_doc";
   String THREAT_FIELD_DEFAULT = "threat:triage:score";


Mime
View raw message