metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmiklav...@apache.org
Subject [09/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
Date Wed, 11 Jul 2018 01:32:25 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 56406f4..7fca764 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -39,8 +39,6 @@ import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.integration.InMemoryComponent;
 import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -472,25 +470,15 @@ public abstract class SearchIntegrationTest {
   @Multiline
   public static String differentTypeFilterQuery;
 
-  protected static IndexDao dao;
   protected static InMemoryComponent indexComponent;
 
-  @Before
-  public synchronized void setup() throws Exception {
-    if(dao == null && indexComponent == null) {
-      indexComponent = startIndex();
-      loadTestData();
-      dao = createDao();
-    }
-  }
-
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 
   @Test
   public void all_query_returns_all_results() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(10, results.size());
@@ -507,7 +495,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void find_one_guid() throws Exception {
     GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
-    Optional<Map<String, Object>> response = dao.getLatestResult(request);
+    Optional<Map<String, Object>> response = getIndexDao().getLatestResult(request);
     Assert.assertTrue(response.isPresent());
     Map<String, Object> doc = response.get();
     Assert.assertEquals("bro", doc.get(getSourceTypeField()));
@@ -519,7 +507,7 @@ public abstract class SearchIntegrationTest {
     List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
     Map<String, Document> docs = new HashMap<>();
 
-    for(Document doc : dao.getAllLatest(request)) {
+    for(Document doc : getIndexDao().getAllLatest(request)) {
       docs.put(doc.getGuid(), doc);
     }
     Assert.assertEquals(2, docs.size());
@@ -532,7 +520,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void filter_query_filters_results() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(3, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
@@ -546,7 +534,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void sort_query_sorts_results_ascending() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     for (int i = 8001; i < 8011; ++i) {
@@ -557,7 +545,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void sort_ascending_with_missing_fields() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(10, results.size());
@@ -575,7 +563,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void sort_descending_with_missing_fields() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(10, results.size());
@@ -593,7 +581,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void results_are_paginated() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(3, results.size());
@@ -608,7 +596,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void returns_results_only_for_specified_indices() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(5, response.getTotal());
     List<SearchResult> results = response.getResults();
     for (int i = 5, j = 0; i > 0; i--, j++) {
@@ -621,7 +609,7 @@ public abstract class SearchIntegrationTest {
   public void facet_query_yields_field_types() throws Exception {
     String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField());
     SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
     Assert.assertEquals(8, facetCounts.size());
@@ -696,14 +684,14 @@ public abstract class SearchIntegrationTest {
   @Test
   public void disabled_facet_query_returns_null_count() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertNull(response.getFacetCounts());
   }
 
   @Test
   public void missing_type_facet_query() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(missingTypeFacetQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
 
     Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
@@ -723,7 +711,7 @@ public abstract class SearchIntegrationTest {
   public void different_type_facet_query() throws Exception {
     thrown.expect(Exception.class);
     SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(3, response.getTotal());
   }
 
@@ -732,14 +720,14 @@ public abstract class SearchIntegrationTest {
     thrown.expect(InvalidSearchException.class);
     thrown.expectMessage("Search result size must be less than 100");
     SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
-    dao.search(request);
+    getIndexDao().search(request);
   }
 
   @Test
   public void column_metadata_for_missing_index() throws Exception {
     // getColumnMetadata with an index that doesn't exist
     {
-      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex"));
+      Map<String, FieldType> fieldTypes = getIndexDao().getColumnMetadata(Collections.singletonList("someindex"));
       Assert.assertEquals(0, fieldTypes.size());
     }
   }
@@ -747,14 +735,14 @@ public abstract class SearchIntegrationTest {
   @Test
   public void no_results_returned_when_query_does_not_match() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(0, response.getTotal());
   }
 
   @Test
   public void group_by_ip_query() throws Exception {
     GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
-    GroupResponse response = dao.group(request);
+    GroupResponse response = getIndexDao().group(request);
 
     // expect only 1 group for 'ip_src_addr'
     Assert.assertEquals("ip_src_addr", response.getGroupedBy());
@@ -778,7 +766,7 @@ public abstract class SearchIntegrationTest {
   public void group_by_returns_results_in_groups() throws Exception {
     // Group by test case, default order is count descending
     GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
-    GroupResponse response = dao.group(request);
+    GroupResponse response = getIndexDao().group(request);
     Assert.assertEquals("is_alert", response.getGroupedBy());
     List<GroupResult> isAlertGroups = response.getGroupResults();
     Assert.assertEquals(2, isAlertGroups.size());
@@ -830,7 +818,7 @@ public abstract class SearchIntegrationTest {
   public void group_by_returns_results_in_sorted_groups() throws Exception {
     // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
     GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
-    GroupResponse response = dao.group(request);
+    GroupResponse response = getIndexDao().group(request);
     Assert.assertEquals("is_alert", response.getGroupedBy());
     List<GroupResult> isAlertGroups = response.getGroupResults();
     Assert.assertEquals(2, isAlertGroups.size());
@@ -909,7 +897,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void queries_fields() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     for (int i = 0; i < 5; ++i) {
@@ -927,7 +915,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public void sort_by_guid() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
-    SearchResponse response = dao.search(request);
+    SearchResponse response = getIndexDao().search(request);
     Assert.assertEquals(5, response.getTotal());
     List<SearchResult> results = response.getResults();
     for (int i = 0; i < 5; ++i) {
@@ -938,7 +926,7 @@ public abstract class SearchIntegrationTest {
   }
 
   @AfterClass
-  public static void stop() throws Exception {
+  public static void stop() {
     indexComponent.stop();
   }
 
@@ -949,9 +937,7 @@ public abstract class SearchIntegrationTest {
   @Test
   public abstract void different_type_filter_query() throws Exception;
 
+  protected abstract IndexDao getIndexDao();
 
-  protected abstract IndexDao createDao() throws Exception;
-  protected abstract InMemoryComponent startIndex() throws Exception;
-  protected abstract void loadTestData() throws Exception;
   protected abstract String getSourceTypeField();
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index 471acf6..eebf0bb 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -20,86 +20,43 @@ import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.hbase.mock.MockHBaseTableProvider;
 import org.apache.metron.hbase.mock.MockHTable;
 import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.indexing.dao.update.ReplaceRequest;
-import org.apache.metron.integration.InMemoryComponent;
-import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-
 public abstract class UpdateIntegrationTest {
 
   private static final int MAX_RETRIES = 10;
   private static final int SLEEP_MS = 500;
-  protected static final String SENSOR_NAME= "test";
-  private static final String TABLE_NAME = "modifications";
+  protected static final String SENSOR_NAME = "test";
   private static final String CF = "p";
-  private static String index;
-  private static MockHTable table;
-  private static IndexDao hbaseDao;
-  private static AccessConfig accessConfig;
 
   protected static MultiIndexDao dao;
-  protected static InMemoryComponent indexComponent;
-
-  @Before
-  public void setup() throws Exception {
-    if(dao == null && indexComponent == null) {
-      index = getIndexName();
-      indexComponent = startIndex();
-      loadTestData();
-      Configuration config = HBaseConfiguration.create();
-      MockHBaseTableProvider tableProvider = new MockHBaseTableProvider();
-      tableProvider.addToCache(TABLE_NAME, CF);
-      table = (MockHTable)tableProvider.getTable(config, TABLE_NAME);
-
-      hbaseDao = new HBaseDao();
-      accessConfig = new AccessConfig();
-      accessConfig.setTableProvider(tableProvider);
-      Map<String, Object> globalConfig = createGlobalConfig();
-      globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME);
-      globalConfig.put(HBaseDao.HBASE_CF, CF);
-      accessConfig.setGlobalConfigSupplier(() -> globalConfig);
-    }
-  }
-
-  protected AccessConfig getAccessConfig() {
-    return accessConfig;
-  }
 
   @Test
   public void test() throws Exception {
-    dao = new MultiIndexDao(hbaseDao, createDao());
-    dao.init(getAccessConfig());
-
     List<Map<String, Object>> inputData = new ArrayList<>();
     for(int i = 0; i < 10;++i) {
       final String name = "message" + i;
       inputData.add(
           new HashMap<String, Object>() {{
-            put("source:type", SENSOR_NAME);
+            put("source.type", SENSOR_NAME);
             put("name" , name);
             put("timestamp", System.currentTimeMillis());
             put(Constants.GUID, name);
           }}
       );
     }
-    addTestData(index, SENSOR_NAME, inputData);
+    addTestData(getIndexName(), SENSOR_NAME, inputData);
     List<Map<String,Object>> docs = null;
     for(int t = 0;t < MAX_RETRIES;++t, Thread.sleep(SLEEP_MS)) {
-      docs = getIndexedTestData(index, SENSOR_NAME);
+      docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
       if(docs.size() >= 10) {
         break;
       }
@@ -115,16 +72,16 @@ public abstract class UpdateIntegrationTest {
         setReplacement(message0);
         setGuid(guid);
         setSensorType(SENSOR_NAME);
-        setIndex(index);
+        setIndex(getIndexName());
       }}, Optional.empty());
 
-      Assert.assertEquals(1, table.size());
+      Assert.assertEquals(1, getMockHTable().size());
       Document doc = dao.getLatest(guid, SENSOR_NAME);
       Assert.assertEquals(message0, doc.getDocument());
       {
         //ensure hbase is up to date
         Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
-        Result r = table.get(g);
+        Result r = getMockHTable().get(g);
         NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
         Assert.assertEquals(1, columns.size());
         Assert.assertEquals(message0
@@ -136,7 +93,7 @@ public abstract class UpdateIntegrationTest {
         //ensure ES is up-to-date
         long cnt = 0;
         for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
-          docs = getIndexedTestData(index, SENSOR_NAME);
+          docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
           cnt = docs
               .stream()
               .filter(d -> message0.get("new-field").equals(d.get("new-field")))
@@ -155,15 +112,15 @@ public abstract class UpdateIntegrationTest {
         setReplacement(message0);
         setGuid(guid);
         setSensorType(SENSOR_NAME);
-        setIndex(index);
+        setIndex(getIndexName());
       }}, Optional.empty());
-      Assert.assertEquals(1, table.size());
+      Assert.assertEquals(1, getMockHTable().size());
       Document doc = dao.getLatest(guid, SENSOR_NAME);
       Assert.assertEquals(message0, doc.getDocument());
       {
         //ensure hbase is up to date
         Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
-        Result r = table.get(g);
+        Result r = getMockHTable().get(g);
         NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
         Assert.assertEquals(2, columns.size());
         Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue())
@@ -177,36 +134,20 @@ public abstract class UpdateIntegrationTest {
         //ensure ES is up-to-date
         long cnt = 0;
         for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t,Thread.sleep(SLEEP_MS)) {
-          docs = getIndexedTestData(index, SENSOR_NAME);
+          docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
           cnt = docs
               .stream()
               .filter(d -> message0.get("new-field").equals(d.get("new-field")))
               .count();
         }
 
-        Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
+        Assert.assertNotEquals("Index is not updated!", cnt, 0);
       }
     }
   }
 
-  @After
-  public void reset() throws Exception {
-    indexComponent.reset();
-  }
-
-  @AfterClass
-  public static void teardown() {
-    if(indexComponent != null) {
-      indexComponent.stop();
-    }
-  }
-
   protected abstract String getIndexName();
-  protected abstract Map<String, Object> createGlobalConfig() throws Exception;
-  protected abstract IndexDao createDao() throws Exception;
-  protected abstract InMemoryComponent startIndex() throws Exception;
-  protected abstract void loadTestData() throws Exception;
+  protected abstract MockHTable getMockHTable();
   protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception;
   protected abstract List<Map<String,Object>> getIndexedTestData(String indexName, String sensorType) throws Exception;
-
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
new file mode 100644
index 0000000..b4f7d38
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
@@ -0,0 +1,1012 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.STATUS_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.Group;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.GroupResult;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.search.SortField;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+public abstract class MetaAlertIntegrationTest {
+
+  private static final String META_INDEX_FLAG = "%META_INDEX%";
+  // To change back after testing
+  protected static int MAX_RETRIES = 10;
+  protected static final int SLEEP_MS = 500;
+  protected static final String SENSOR_NAME = "test";
+
+  protected static final String NEW_FIELD = "new-field";
+  protected static final String NAME_FIELD = "name";
+  protected static final String DATE_FORMAT = "yyyy.MM.dd.HH";
+
+  // Separate the raw indices from the query indices. ES for example, modifies the indices to
+  // have a separator
+  protected ArrayList<String> allIndices = new ArrayList<String>() {
+    {
+      add(getTestIndexName());
+      add(getMetaAlertIndex());
+    }
+  };
+
+  protected ArrayList<String> queryIndices = allIndices;
+
+  protected static MetaAlertDao metaDao;
+
+  /**
+   {
+   "guid": "meta_alert",
+   "index": "%META_INDEX%",
+   "patch": [
+   {
+   "op": "add",
+   "path": "/name",
+   "value": "New Meta Alert"
+   }
+   ],
+   "sensorType": "metaalert"
+   }
+   */
+  @Multiline
+  public static String namePatchRequest;
+
+  /**
+   {
+   "guid": "meta_alert",
+   "index": "%META_INDEX%",
+   "patch": [
+   {
+   "op": "add",
+   "path": "/name",
+   "value": "New Meta Alert"
+   },
+   {
+   "op": "add",
+   "path": "/alert",
+   "value": []
+   }
+   ],
+   "sensorType": "metaalert"
+   }
+   */
+  @Multiline
+  public static String alertPatchRequest;
+
+  /**
+   {
+   "guid": "meta_alert",
+   "index": "%META_INDEX%",
+   "patch": [
+   {
+   "op": "add",
+   "path": "/status",
+   "value": "inactive"
+   },
+   {
+   "op": "add",
+   "path": "/name",
+   "value": "New Meta Alert"
+   }
+   ],
+   "sensorType": "metaalert"
+   }
+   */
+  @Multiline
+  public static String statusPatchRequest;
+
+
+  @Test
+  public void shouldGetAllMetaAlertsForAlert() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(3);
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlerts
+    List<Map<String, Object>> metaAlerts = buildMetaAlerts(12, MetaAlertStatus.ACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    metaAlerts.add(buildMetaAlert("meta_active_12", MetaAlertStatus.ACTIVE,
+        Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
+    metaAlerts.add(buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE,
+        Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    addRecords(metaAlerts, getMetaAlertIndex(), METAALERT_TYPE);
+
+    // Verify load was successful
+    List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert ->
+        new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE))
+        .collect(Collectors.toList());
+    createdDocs.addAll(alerts.stream().map(alert ->
+        new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME))
+        .collect(Collectors.toList()));
+    findCreatedDocs(createdDocs);
+
+    {
+      // Verify searches successfully return more than 10 results
+      SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0");
+      List<SearchResult> searchResults0 = searchResponse0.getResults();
+      Assert.assertEquals(13, searchResults0.size());
+      Set<Map<String, Object>> resultSet = new HashSet<>();
+      Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource()));
+      StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n");
+      reason.append(Joiner.on("\n").join(resultSet));
+      Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0)));
+
+      // Verify no meta alerts are returned because message_1 was not added to any
+      SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1");
+      List<SearchResult> searchResults1 = searchResponse1.getResults();
+      Assert.assertEquals(0, searchResults1.size());
+
+      // Verify only the meta alert message_2 was added to is returned
+      SearchResponse searchResponse2 = metaDao.getAllMetaAlertsForAlert("message_2");
+      List<SearchResult> searchResults2 = searchResponse2.getResults();
+      Assert.assertEquals(1, searchResults2.size());
+      Assert.assertEquals(metaAlerts.get(12), searchResults2.get(0).getSource());
+    }
+  }
+
+  @Test
+  public void getAllMetaAlertsForAlertShouldThrowExceptionForEmptyGuid() throws Exception {
+    try {
+      metaDao.getAllMetaAlertsForAlert("");
+      Assert.fail("An exception should be thrown for empty guid");
+    } catch (InvalidSearchException ise) {
+      Assert.assertEquals("Guid cannot be empty", ise.getMessage());
+    }
+  }
+
+  @Test
+  public void shouldCreateMetaAlert() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(3);
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("message_2", SENSOR_NAME)));
+
+    {
+      MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{
+        setAlerts(new ArrayList<GetRequest>() {{
+          add(new GetRequest("message_1", SENSOR_NAME));
+          add(new GetRequest("message_2", SENSOR_NAME, getTestIndexFullName()));
+        }});
+        setGroups(Collections.singletonList("group"));
+      }};
+      MetaAlertCreateResponse metaAlertCreateResponse = metaDao
+          .createMetaAlert(metaAlertCreateRequest);
+      {
+        // Verify metaAlert was created
+        findCreatedDoc(metaAlertCreateResponse.getGuid(), METAALERT_TYPE);
+      }
+      {
+        // Verify alert 0 was not updated with metaalert field
+        Document alert = metaDao.getLatest("message_0", SENSOR_NAME);
+        Assert.assertEquals(4, alert.getDocument().size());
+        Assert.assertNull(alert.getDocument().get(METAALERT_FIELD));
+      }
+      {
+        // Verify alert 1 was properly updated with metaalert field
+        Map<String, Object> expectedAlert = new HashMap<>(alerts.get(1));
+        expectedAlert
+            .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+        findUpdatedDoc(expectedAlert, "message_1", SENSOR_NAME);
+      }
+      {
+        // Verify alert 2 was properly updated with metaalert field
+        Map<String, Object> expectedAlert = new HashMap<>(alerts.get(2));
+        expectedAlert
+            .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+        findUpdatedDoc(expectedAlert, "message_2", SENSOR_NAME);
+      }
+    }
+  }
+
+  @Test
+  public void shouldAddAlertsToMetaAlert() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(4);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlert
+    Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("message_2", SENSOR_NAME),
+        new GetRequest("message_3", SENSOR_NAME),
+        new GetRequest("meta_alert", METAALERT_TYPE)
+    ));
+
+    // Build expected metaAlert after alerts are added
+    Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+
+    // Verify the proper alerts were added
+    @SuppressWarnings("unchecked")
+    List<Map<String, Object>> metaAlertAlerts = new ArrayList<>(
+        (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+    // Alert 0 is already in the metaalert. Add alerts 1 and 2.
+    Map<String, Object> expectedAlert1 = alerts.get(1);
+    expectedAlert1.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    metaAlertAlerts.add(expectedAlert1);
+    Map<String, Object> expectedAlert2 = alerts.get(2);
+    expectedAlert2.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    metaAlertAlerts.add(expectedAlert2);
+    expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+    // Verify the counts were properly updated
+    expectedMetaAlert.put("average", 1.0d);
+    expectedMetaAlert.put("min", 0.0d);
+    expectedMetaAlert.put("median", 1.0d);
+    expectedMetaAlert.put("max", 2.0d);
+    expectedMetaAlert.put("count", 3);
+    expectedMetaAlert.put("sum", 3.0d);
+    expectedMetaAlert.put(getThreatTriageField(), 3.0d);
+
+    {
+      // Verify alerts were successfully added to the meta alert
+      Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_1", SENSOR_NAME),
+              new GetRequest("message_2", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify False when alerts are already in a meta alert and no new alerts are added
+      Assert.assertFalse(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_0", SENSOR_NAME),
+              new GetRequest("message_1", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify only 1 alert is added when a list of alerts only contains 1 alert that is not in the meta alert
+      metaAlertAlerts = (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD);
+      Map<String, Object> expectedAlert3 = alerts.get(3);
+      expectedAlert3.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+      metaAlertAlerts.add(expectedAlert3);
+      expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+      expectedMetaAlert.put("average", 1.5d);
+      expectedMetaAlert.put("min", 0.0d);
+      expectedMetaAlert.put("median", 1.5d);
+      expectedMetaAlert.put("max", 3.0d);
+      expectedMetaAlert.put("count", 4);
+      expectedMetaAlert.put("sum", 6.0d);
+      expectedMetaAlert.put(getThreatTriageField(), 6.0d);
+
+      Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_2", SENSOR_NAME),
+              new GetRequest("message_3", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void shouldRemoveAlertsFromMetaAlert() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(4);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    alerts.get(2).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    alerts.get(3).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlert
+    Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+        Optional.of(Arrays.asList(alerts.get(0), alerts.get(1), alerts.get(2), alerts.get(3))));
+    addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("message_2", SENSOR_NAME),
+        new GetRequest("message_3", SENSOR_NAME),
+        new GetRequest("meta_alert", METAALERT_TYPE)));
+
+    // Build expected metaAlert after alerts are added
+    Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+
+    // Verify the proper alerts were added
+    List<Map<String, Object>> metaAlertAlerts = new ArrayList<>(
+        (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+    metaAlertAlerts.remove(0);
+    metaAlertAlerts.remove(0);
+    expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+    // Verify the counts were properly updated
+    expectedMetaAlert.put("average", 2.5d);
+    expectedMetaAlert.put("min", 2.0d);
+    expectedMetaAlert.put("median", 2.5d);
+    expectedMetaAlert.put("max", 3.0d);
+    expectedMetaAlert.put("count", 2);
+    expectedMetaAlert.put("sum", 5.0d);
+    expectedMetaAlert.put(getThreatTriageField(), 5.0d);
+
+    {
+      // Verify a list of alerts are removed from a meta alert
+      Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_0", SENSOR_NAME),
+              new GetRequest("message_1", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify False when alerts are not present in a meta alert and no alerts are removed
+      Assert.assertFalse(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_0", SENSOR_NAME),
+              new GetRequest("message_1", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify only 1 alert is removed when a list of alerts only contains 1 alert that is in the meta alert
+      metaAlertAlerts = new ArrayList<>(
+          (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+      metaAlertAlerts.remove(0);
+      expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+      expectedMetaAlert.put("average", 3.0d);
+      expectedMetaAlert.put("min", 3.0d);
+      expectedMetaAlert.put("median", 3.0d);
+      expectedMetaAlert.put("max", 3.0d);
+      expectedMetaAlert.put("count", 1);
+      expectedMetaAlert.put("sum", 3.0d);
+      expectedMetaAlert.put(getThreatTriageField(), 3.0d);
+
+      Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+          .asList(new GetRequest("message_0", SENSOR_NAME),
+              new GetRequest("message_2", SENSOR_NAME))));
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify all alerts are removed from a metaAlert
+      metaAlertAlerts = new ArrayList<>(
+          (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+      metaAlertAlerts.remove(0);
+      if (isEmptyMetaAlertList()) {
+        expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+      } else {
+        expectedMetaAlert.remove(ALERT_FIELD);
+      }
+
+      expectedMetaAlert.put("average", 0.0d);
+      expectedMetaAlert.put("count", 0);
+      expectedMetaAlert.put("sum", 0.0d);
+      expectedMetaAlert.put(getThreatTriageField(), 0.0d);
+
+      // Handle the cases with non-finite Double values on a per store basis
+      if (isFiniteDoubleOnly()) {
+        expectedMetaAlert.put("min", String.valueOf(Double.POSITIVE_INFINITY));
+        expectedMetaAlert.put("median", String.valueOf(Double.NaN));
+        expectedMetaAlert.put("max", String.valueOf(Double.NEGATIVE_INFINITY));
+      } else {
+        expectedMetaAlert.put("min", Double.POSITIVE_INFINITY);
+        expectedMetaAlert.put("median", Double.NaN);
+        expectedMetaAlert.put("max", Double.NEGATIVE_INFINITY);
+      }
+
+      // Verify removing alerts cannot result in an empty meta alert
+      try {
+        metaDao.removeAlertsFromMetaAlert("meta_alert",
+                Collections.singletonList(new GetRequest("message_3", SENSOR_NAME)));
+        Assert.fail("Removing these alerts will result in an empty meta alert.  Empty meta alerts are not allowed.");
+      } catch (IllegalStateException ise) {
+        Assert.assertEquals("Removing these alerts will result in an empty meta alert.  Empty meta alerts are not allowed.",
+                ise.getMessage());
+      }
+    }
+  }
+
+  @Test
+  public void addRemoveAlertsShouldThrowExceptionForInactiveMetaAlert() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(2);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlert
+    Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.INACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("meta_alert", METAALERT_TYPE)));
+
+    {
+      // Verify alerts cannot be added to an INACTIVE meta alert
+      try {
+        metaDao.addAlertsToMetaAlert("meta_alert",
+            Collections.singletonList(new GetRequest("message_1", SENSOR_NAME)));
+        Assert.fail("Adding alerts to an inactive meta alert should throw an exception");
+      } catch (IllegalStateException ise) {
+        Assert.assertEquals("Adding alerts to an INACTIVE meta alert is not allowed",
+            ise.getMessage());
+      }
+    }
+
+    {
+      // Verify alerts cannot be removed from an INACTIVE meta alert
+      try {
+        metaDao.removeAlertsFromMetaAlert("meta_alert",
+            Collections.singletonList(new GetRequest("message_0", SENSOR_NAME)));
+        Assert.fail("Removing alerts from an inactive meta alert should throw an exception");
+      } catch (IllegalStateException ise) {
+        Assert.assertEquals("Removing alerts from an INACTIVE meta alert is not allowed",
+            ise.getMessage());
+      }
+    }
+  }
+
+  @Test
+  public void shouldUpdateMetaAlertStatus() throws Exception {
+    int numChildAlerts = 25;
+    int numUnrelatedAlerts = 25;
+    int totalAlerts = numChildAlerts + numUnrelatedAlerts;
+
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(totalAlerts);
+    List<Map<String, Object>> childAlerts = alerts.subList(0, numChildAlerts);
+    List<Map<String, Object>> unrelatedAlerts = alerts.subList(numChildAlerts, totalAlerts);
+    for (Map<String, Object> alert : childAlerts) {
+      alert.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+    }
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlerts
+    Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+        Optional.of(childAlerts));
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(),
+        METAALERT_TYPE);
+
+    List<GetRequest> requests = new ArrayList<>();
+    for (int i = 0; i < numChildAlerts; ++i) {
+      requests.add(new GetRequest("message_" + i, SENSOR_NAME));
+    }
+    requests.add(new GetRequest("meta_alert", METAALERT_TYPE));
+
+    // Verify load was successful
+    findCreatedDocs(requests);
+
+    {
+      // Verify status changed to inactive and child alerts are updated
+      Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.INACTIVE));
+
+      Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+      expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString());
+
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+      for (int i = 0; i < numChildAlerts; ++i) {
+        Map<String, Object> expectedAlert = new HashMap<>(childAlerts.get(i));
+        setEmptiedMetaAlertField(expectedAlert);
+        findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+      }
+
+      // Ensure unrelated alerts are unaffected
+      for (int i = 0; i < numUnrelatedAlerts; ++i) {
+        Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+        // Make sure to handle the guid offset from creation
+        findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+      }
+    }
+
+    {
+      // Verify status changed to active and child alerts are updated
+      Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
+
+      Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+      expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+      for (int i = 0; i < numChildAlerts; ++i) {
+        Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i));
+        expectedAlert.put("metaalerts", Collections.singletonList("meta_alert"));
+        findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+      }
+
+      // Ensure unrelated alerts are unaffected
+      for (int i = 0; i < numUnrelatedAlerts; ++i) {
+        Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+        // Make sure to handle the guid offset from creation
+        findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+      }
+
+      {
+        // Verify status changed to current status has no effect
+        Assert.assertFalse(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
+
+        findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+        for (int i = 0; i < numChildAlerts; ++i) {
+          Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i));
+          expectedAlert.put("metaalerts", Collections.singletonList("meta_alert"));
+          findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+        }
+
+        // Ensure unrelated alerts are unaffected
+        for (int i = 0; i < numUnrelatedAlerts; ++i) {
+          Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+          // Make sure to handle the guid offset from creation
+          findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void shouldSearchByStatus() throws Exception {
+    // Load alert
+    List<Map<String, Object>> alerts = buildAlerts(1);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+    alerts.get(0).put("ip_src_addr", "192.168.1.1");
+    alerts.get(0).put("ip_src_port", 8010);
+
+    // Load metaAlerts
+    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive",
+        MetaAlertStatus.INACTIVE,
+        Optional.empty());
+
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(),
+        METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("meta_active", METAALERT_TYPE),
+        new GetRequest("meta_inactive", METAALERT_TYPE)));
+
+    SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+      {
+        setQuery("*:*");
+        setIndices(Collections.singletonList(METAALERT_TYPE));
+        setFrom(0);
+        setSize(5);
+        setSort(Collections.singletonList(new SortField() {{
+          setField(Constants.GUID);
+        }}));
+      }
+    });
+
+    // Verify only active meta alerts are returned
+    Assert.assertEquals(1, searchResponse.getTotal());
+    Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(),
+        searchResponse.getResults().get(0).getSource().get(STATUS_FIELD));
+  }
+
+
+  @Test
+  public void shouldHidesAlertsOnGroup() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(2);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+    alerts.get(0).put("ip_src_addr", "192.168.1.1");
+    alerts.get(0).put("score", 1);
+    alerts.get(1).put("ip_src_addr", "192.168.1.1");
+    alerts.get(1).put("score", 10);
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Put the nested type into the test index, so that it'll match appropriately
+    setupTypings();
+
+    // Don't need any meta alerts to actually exist, since we've populated the field on the alerts.
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME)));
+
+    // Build our group request
+    Group searchGroup = new Group();
+    searchGroup.setField("ip_src_addr");
+    List<Group> groupList = new ArrayList<>();
+    groupList.add(searchGroup);
+    GroupResponse groupResponse = metaDao.group(new GroupRequest() {
+      {
+        setQuery("ip_src_addr:192.168.1.1");
+        setIndices(queryIndices);
+        setScoreField("score");
+        setGroups(groupList);
+      }
+    });
+
+    // Should only return the standalone alert in the group
+    GroupResult result = groupResponse.getGroupResults().get(0);
+    Assert.assertEquals(1, result.getTotal());
+    Assert.assertEquals("192.168.1.1", result.getKey());
+    // No delta, since no ops happen
+    Assert.assertEquals(10.0d, result.getScore(), 0.0d);
+  }
+
+  // This test is important enough that everyone should implement it, but is pretty specific to
+  // implementation
+  @Test
+  public abstract void shouldSearchByNestedAlert() throws Exception;
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void shouldUpdateMetaAlertOnAlertUpdate() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(2);
+    alerts.get(0).put(METAALERT_FIELD, Arrays.asList("meta_active", "meta_inactive"));
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Load metaAlerts
+    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive",
+        MetaAlertStatus.INACTIVE,
+        Optional.of(Collections.singletonList(alerts.get(0))));
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(),
+        METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("meta_active", METAALERT_TYPE),
+        new GetRequest("meta_inactive", METAALERT_TYPE)));
+
+    {
+      // Modify the first message and add a new field
+      Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) {
+        {
+          put(NEW_FIELD, "metron");
+          put(THREAT_FIELD_DEFAULT, 10.0d);
+        }
+      };
+      String guid = "" + message0.get(Constants.GUID);
+      metaDao.update(new Document(message0, guid, SENSOR_NAME, null),
+          Optional.of(getTestIndexFullName()));
+
+      {
+        // Verify alerts are up-to-date
+        findUpdatedDoc(message0, guid, SENSOR_NAME);
+        long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD));
+        if (cnt == 0) {
+          Assert.fail("Alert not updated!");
+        }
+      }
+
+      {
+        // Verify meta alerts are up-to-date
+        long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron");
+        if (cnt == 0) {
+          Assert.fail("Active metaalert was not updated!");
+        }
+        if (cnt != 1) {
+          Assert.fail("Metaalerts not updated correctly!");
+        }
+      }
+    }
+    //modify the same message and modify the new field
+    {
+      Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) {
+        {
+          put(NEW_FIELD, "metron2");
+        }
+      };
+      String guid = "" + message0.get(Constants.GUID);
+      metaDao.update(new Document(message0, guid, SENSOR_NAME, null), Optional.empty());
+
+      {
+        // Verify index is up-to-date
+        findUpdatedDoc(message0, guid, SENSOR_NAME);
+        long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD));
+        if (cnt == 0) {
+          Assert.fail("Alert not updated!");
+        }
+      }
+      {
+        // Verify meta alerts are up-to-date
+        long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron2");
+        if (cnt == 0) {
+          Assert.fail("Active metaalert was not updated!");
+        }
+        if (cnt != 1) {
+          Assert.fail("Metaalerts not updated correctly!");
+        }
+      }
+    }
+  }
+
+  @Test
+  public void shouldThrowExceptionOnMetaAlertUpdate() throws Exception {
+    Document metaAlert = new Document(new HashMap<>(), "meta_alert", METAALERT_TYPE, 0L);
+    try {
+      // Verify a meta alert cannot be updated in the meta alert dao
+      metaDao.update(metaAlert, Optional.empty());
+      Assert.fail("Direct meta alert update should throw an exception");
+    } catch (UnsupportedOperationException uoe) {
+      Assert.assertEquals("Meta alerts cannot be directly updated", uoe.getMessage());
+    }
+  }
+
+  @Test
+  public void shouldPatchAllowedMetaAlerts() throws Exception {
+    // Load alerts
+    List<Map<String, Object>> alerts = buildAlerts(2);
+    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+    alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+    addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+    // Put the nested type into the test index, so that it'll match appropriately
+    setupTypings();
+
+    // Load metaAlerts
+    Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+        Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
+    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+    addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+    // Verify load was successful
+    findCreatedDocs(Arrays.asList(
+        new GetRequest("message_0", SENSOR_NAME),
+        new GetRequest("message_1", SENSOR_NAME),
+        new GetRequest("meta_alert", METAALERT_TYPE)));
+
+    Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+    expectedMetaAlert.put(NAME_FIELD, "New Meta Alert");
+    {
+      // Verify a patch to a field other than "status" or "alert" can be patched
+      String namePatch = namePatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex());
+      PatchRequest patchRequest = JSONUtils.INSTANCE.load(namePatch, PatchRequest.class);
+      metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify a patch to an alert field should throw an exception
+      try {
+        String alertPatch = alertPatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex());
+        PatchRequest patchRequest = JSONUtils.INSTANCE.load(alertPatch, PatchRequest.class);
+        metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+        Assert.fail("A patch on the alert field should throw an exception");
+      } catch (IllegalArgumentException iae) {
+        Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths.  "
+                + "Please use the add/remove alert or update status functions instead.",
+            iae.getMessage());
+      }
+
+      // Verify the metaAlert was not updated
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+
+    {
+      // Verify a patch to a status field should throw an exception
+      try {
+        String statusPatch = statusPatchRequest
+            .replace(META_INDEX_FLAG, getMetaAlertIndex());
+        PatchRequest patchRequest = JSONUtils.INSTANCE.load(statusPatch, PatchRequest.class);
+        metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+        Assert.fail("A patch on the status field should throw an exception");
+      } catch (IllegalArgumentException iae) {
+        Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths.  "
+                + "Please use the add/remove alert or update status functions instead.",
+            iae.getMessage());
+      }
+
+      // Verify the metaAlert was not updated
+      findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+    }
+  }
+
+  protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
+      throws InterruptedException, IOException, OriginalNotFoundException {
+    commit();
+    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+      Document doc = metaDao.getLatest(guid, sensorType);
+      // Change the underlying document alerts lists to sets to avoid ordering issues.
+      convertAlertsFieldToSet(doc.getDocument());
+      convertAlertsFieldToSet(message0);
+
+      if (doc.getDocument() != null && message0.equals(doc.getDocument())) {
+        convertAlertsFieldToList(doc.getDocument());
+        convertAlertsFieldToList(message0);
+        return;
+      }
+    }
+
+    throw new OriginalNotFoundException(
+        "Count not find " + guid + " after " + MAX_RETRIES + " tries");
+  }
+
+  protected void convertAlertsFieldToSet(Map<String, Object> document) {
+    if (document.get(ALERT_FIELD) instanceof List) {
+      @SuppressWarnings("unchecked")
+      List<Map<String, Object>> message0AlertField = (List<Map<String, Object>>) document
+          .get(ALERT_FIELD);
+      Set<Map<String, Object>> message0AlertSet = new HashSet<>(message0AlertField);
+      document.put(ALERT_FIELD, message0AlertSet);
+    }
+  }
+
+  protected void convertAlertsFieldToList(Map<String, Object> document) {
+    if (document.get(ALERT_FIELD) instanceof Set) {
+      @SuppressWarnings("unchecked")
+      Set<Map<String, Object>> message0AlertField = (Set<Map<String, Object>>) document
+          .get(ALERT_FIELD);
+      List<Map<String, Object>> message0AlertList = new ArrayList<>(message0AlertField);
+      message0AlertList.sort(Comparator.comparing(o -> ((String) o.get(Constants.GUID))));
+      document.put(ALERT_FIELD, message0AlertList);
+    }
+  }
+
+  protected boolean findCreatedDoc(String guid, String sensorType)
+      throws InterruptedException, IOException, OriginalNotFoundException {
+    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+      Document doc = metaDao.getLatest(guid, sensorType);
+      if (doc != null) {
+        return true;
+      }
+    }
+    throw new OriginalNotFoundException(
+        "Count not find " + guid + " after " + MAX_RETRIES + "tries");
+  }
+
+  protected boolean findCreatedDocs(List<GetRequest> getRequests)
+      throws InterruptedException, IOException, OriginalNotFoundException {
+    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+      Iterable<Document> docs = metaDao.getAllLatest(getRequests);
+      if (docs != null) {
+        int docCount = 0;
+        for (Document doc : docs) {
+          docCount++;
+        }
+        if (getRequests.size() == docCount) {
+          return true;
+        }
+      }
+    }
+    throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries");
+  }
+
+  protected List<Map<String, Object>> buildAlerts(int count) {
+    List<Map<String, Object>> inputData = new ArrayList<>();
+    for (int i = 0; i < count; ++i) {
+      final String guid = "message_" + i;
+      Map<String, Object> alerts = new HashMap<>();
+      alerts.put(Constants.GUID, guid);
+      alerts.put(getSourceTypeField(), SENSOR_NAME);
+      alerts.put(THREAT_FIELD_DEFAULT, (double) i);
+      alerts.put("timestamp", System.currentTimeMillis());
+      inputData.add(alerts);
+    }
+    return inputData;
+  }
+
+  protected List<Map<String, Object>> buildMetaAlerts(int count, MetaAlertStatus status,
+      Optional<List<Map<String, Object>>> alerts) {
+    List<Map<String, Object>> inputData = new ArrayList<>();
+    for (int i = 0; i < count; ++i) {
+      final String guid = "meta_" + status.getStatusString() + "_" + i;
+      inputData.add(buildMetaAlert(guid, status, alerts));
+    }
+    return inputData;
+  }
+
+  protected Map<String, Object> buildMetaAlert(String guid, MetaAlertStatus status,
+      Optional<List<Map<String, Object>>> alerts) {
+    Map<String, Object> metaAlert = new HashMap<>();
+    metaAlert.put(Constants.GUID, guid);
+    metaAlert.put(getSourceTypeField(), METAALERT_TYPE);
+    metaAlert.put(STATUS_FIELD, status.getStatusString());
+    if (alerts.isPresent()) {
+      List<Map<String, Object>> alertsList = alerts.get();
+      metaAlert.put(ALERT_FIELD, alertsList);
+    }
+    return metaAlert;
+  }
+
+  protected abstract long getMatchingAlertCount(String fieldName, Object fieldValue)
+      throws IOException, InterruptedException;
+
+  protected abstract void addRecords(List<Map<String, Object>> inputData, String index,
+      String docType) throws IOException;
+
+  protected abstract long getMatchingMetaAlertCount(String fieldName, String fieldValue)
+      throws IOException, InterruptedException;
+
+  protected abstract void setupTypings();
+
+  // Get the base index name without any adjustments (e.g. without ES's "_index")
+  protected abstract String getTestIndexName();
+
+  // Get the full name of the test index.  E.g. Elasticsearch appends "_index"
+  protected String getTestIndexFullName() {
+    return getTestIndexName();
+  }
+
+  protected abstract String getMetaAlertIndex();
+
+  protected abstract String getSourceTypeField();
+
+  protected String getThreatTriageField() {
+    return THREAT_FIELD_DEFAULT;
+  }
+
+  // Allow for impls to do any commit they need to do.
+  protected void commit() throws IOException {
+  }
+
+  // Different stores can have different representations of empty metaalerts field.
+  // E.g. Solr expects the field to not be present, ES expects it to be empty.
+  protected abstract void setEmptiedMetaAlertField(Map<String, Object> docMap);
+
+  // Different stores may choose to store non finite double values as Strings.
+  // E.g. NaN may be a string, not a double value.
+  protected abstract boolean isFiniteDoubleOnly();
+
+  // Different stores may choose to return empty alerts lists differently.
+  // E.g. It may be missing completely, or may be an empty list
+  protected abstract boolean isEmptyMetaAlertList();
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
new file mode 100644
index 0000000..1359ba9
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_SORT_DEFAULT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.indexing.dao.update.Document;
+import org.junit.Test;
+
+public class MetaScoresTest {
+  @Test
+  public void testCalculateMetaScoresList() {
+    final double delta = 0.001;
+    List<Map<String, Object>> alertList = new ArrayList<>();
+
+    // add an alert with a threat score
+    alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 10.0f));
+
+    // add a second alert with a threat score
+    alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 20.0f));
+
+    // add a third alert with NO threat score
+    alertList.add(Collections.singletonMap("alert3", "has no threat score"));
+
+    // create the metaalert
+    Map<String, Object> docMap = new HashMap<>();
+    docMap.put(ALERT_FIELD, alertList);
+    Document metaalert = new Document(docMap, "guid", METAALERT_TYPE, 0L);
+
+    // calculate the threat score for the metaalert
+    MetaScores.calculateMetaScores(metaalert, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT);
+
+    // the metaalert must contain a summary of all child threat scores
+    assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta);
+    assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta);
+    assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta);
+    assertEquals(2L, metaalert.getDocument().get("count"));
+    assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta);
+    assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta);
+
+    // it must contain an overall threat score; a float to match the type of the threat score of
+    // the other sensor indices
+    Object threatScore = metaalert.getDocument().get(THREAT_FIELD_DEFAULT);
+    assertTrue(threatScore instanceof Float);
+
+    // by default, the overall threat score is the sum of all child threat scores
+    assertEquals(30.0F, threatScore);
+  }
+}


Mime
View raw message