metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject [04/50] [abbrv] metron git commit: METRON-1429 SearchIntegrationTest refactor (merrimanr) closes apache/metron#909
Date Thu, 22 Feb 2018 20:14:15 GMT
METRON-1429 SearchIntegrationTest refactor (merrimanr) closes apache/metron#909


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/35d81cb9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/35d81cb9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/35d81cb9

Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 35d81cb9d1e7cac9d13b64bc3baaf6f6b3354d82
Parents: acb8b92
Author: merrimanr <merrimanr@gmail.com>
Authored: Thu Jan 25 10:30:06 2018 -0600
Committer: merrimanr <merrimanr@apache.org>
Committed: Thu Jan 25 10:30:06 2018 -0600

----------------------------------------------------------------------
 .../MetaAlertControllerIntegrationTest.java     |  11 +-
 .../UpdateControllerIntegrationTest.java        |   2 +-
 .../ElasticsearchSearchIntegrationTest.java     |  99 +-----
 .../indexing/dao/SearchIntegrationTest.java     | 345 +++++--------------
 4 files changed, 94 insertions(+), 363 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
index b0dd774..3e69e37 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
@@ -96,13 +96,22 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest
{
   @Multiline
   public static String create;
 
+  /**
+   * [
+   *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
+   *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
+   * ]
+   */
+  @Multiline
+  public static String metaAlertData;
+
   @Before
   public void setup() throws Exception {
     this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
     ImmutableMap<String, String> testData = ImmutableMap.of(
         "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
         "snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
-        MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+        MetaAlertDao.METAALERTS_INDEX, metaAlertData
     );
     loadTestData(testData);
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index 57a1b28..e8d00d3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -121,7 +121,7 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest
{
     ImmutableMap<String, String> testData = ImmutableMap.of(
         "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
         "snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
-        MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+        MetaAlertDao.METAALERTS_INDEX, MetaAlertControllerIntegrationTest.metaAlertData
     );
     loadTestData(testData);
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 3949c6d..f86a04d 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -19,35 +19,25 @@ package org.apache.metron.elasticsearch.integration;
 
 
 import java.io.File;
-import java.util.HashMap;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.concurrent.ExecutionException;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
 import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
 import org.apache.metron.indexing.dao.AccessConfig;
 import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.MetaAlertDao;
 import org.apache.metron.indexing.dao.SearchIntegrationTest;
 import org.apache.metron.integration.InMemoryComponent;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.ExecutionException;
-
 public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
 
   private static String indexDir = "target/elasticsearch_search";
@@ -181,34 +171,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
   @Multiline
   private static String broDefaultStringMappings;
 
-  /**
-   * {
-   * "metaalert_doc": {
-   *   "properties": {
-   *     "guid": { "type": "keyword" },
-   *     "alert": {
-   *        "type": "nested",
-   *        "properties": {
-   *          "guid": { "type": "keyword" }
-   *        }
-   *     },
-   *     "average": { "type": "keyword" },
-   *     "min" : { "type": "keyword" },
-   *     "median" : { "type": "keyword" },
-   *     "max": { "type": "keyword" },
-   *     "count": { "type": "keyword" },
-   *     "sum": { "type": "keyword" },
-   *     "source:type": {
-   *       "type": "text",
-   *       "fielddata" : "true"
-   *                    }
-   *   }
-   * }
-   * }
-   */
-  @Multiline
-  private static String metaAlertTypeMappings;
-
   @Override
   protected IndexDao createDao() throws Exception {
     AccessConfig config = new AccessConfig();
@@ -246,14 +208,13 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
             .addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get();
     es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
             .addMapping("snort_doc", snortTypeMappings).get();
-    es.getClient().admin().indices().prepareCreate(MetaAlertDao.METAALERTS_INDEX)
-        .addMapping(MetaAlertDao.METAALERT_DOC, metaAlertTypeMappings).get();
 
     BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
     JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
     for(Object o: broArray) {
       JSONObject jsonObject = (JSONObject) o;
       IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("bro_index_2017.01.01.01",
"bro_doc");
+      indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
       indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
       indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
       bulkRequest.add(indexRequestBuilder);
@@ -262,68 +223,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
     for(Object o: snortArray) {
       JSONObject jsonObject = (JSONObject) o;
       IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("snort_index_2017.01.01.02",
"snort_doc");
+      indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
       indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
       indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
       bulkRequest.add(indexRequestBuilder);
     }
-    JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData);
-    for(Object o: metaAlertArray) {
-      JSONObject jsonObject = (JSONObject) o;
-      IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalert_index",
"metaalert_doc");
-      indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
-      bulkRequest.add(indexRequestBuilder);
-    }
     BulkResponse bulkResponse = bulkRequest.execute().actionGet();
     if (bulkResponse.hasFailures()) {
       throw new RuntimeException("Failed to index test data");
     }
-
-    SearchResponse broDocs = es.getClient()
-        .prepareSearch("bro_index_2017.01.01.01")
-        .setTypes("bro_doc")
-        .setQuery(QueryBuilders.matchAllQuery())
-        .get();
-    // We're changing the _id field, we need to create a copy and delete the original.
-    for (SearchHit hit : broDocs.getHits()) {
-      // Bro GUIDs to collide while using the standard analyzer
-      // Use timestamp as part of guid because query may not return in order each time
-      IndexRequest indexRequest = new IndexRequest()
-          .index("bro_index_2017.01.01.01")
-          .type("bro_doc")
-          .id("bro-" + hit.getSource().get("timestamp"))
-          .source(hit.getSource());
-      es.getClient().index(indexRequest).get();
-
-      // Delete the original
-      es.getClient()
-          .prepareDelete("bro_index_2017.01.01.01", "bro_doc", hit.getId())
-          .get();
-    }
-
-    // Wait until everything is updated
-    // Assume true until proven otherwise.
-    boolean allUpdated = true;
-    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
-      allUpdated = true;
-      SearchResponse response = es.getClient()
-          .prepareSearch("bro_index_2017.01.01.01")
-          .setTypes("bro_doc")
-          .setQuery(QueryBuilders.matchAllQuery())
-          .get();
-      if (response.getHits().getTotalHits() == 0) {
-        throw new IllegalStateException("Bro index is empty. No docs to validate were updated");
-      }
-      for (SearchHit hit : response.getHits()) {
-        if (!hit.getId().startsWith("bro-")) {
-          allUpdated = false;
-        }
-      }
-      if (allUpdated) {
-        break;
-      }
-    }
-    if (!allUpdated) {
-      throw new IllegalStateException("Unable to update Elasticsearch ids properly");
-    }
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 72e632f..f381688 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -46,14 +46,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
 public abstract class SearchIntegrationTest {
   /**
    * [
@@ -80,15 +72,6 @@ public abstract class SearchIntegrationTest {
   public static String snortData;
 
   /**
-   * [
-   *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
-   *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
-   * ]
-   */
-  @Multiline
-  public static String metaAlertData;
-
-  /**
    * {
    * "indices": ["bro", "snort"],
    * "query": "*",
@@ -107,7 +90,7 @@ public abstract class SearchIntegrationTest {
 
   /**
    * {
-   * "guid": "bro-3",
+   * "guid": "bro_3",
    * "sensorType": "bro"
    * }
    */
@@ -117,12 +100,12 @@ public abstract class SearchIntegrationTest {
   /**
    * [
    * {
-   * "guid": "bro-1",
+   * "guid": "bro_1",
    * "sensorType": "bro"
    * },
    * {
-   * "guid": "bro-2",
-   * "sensorType": "bro"
+   * "guid": "snort_2",
+   * "sensorType": "snort"
    * }
    * ]
    */
@@ -240,7 +223,7 @@ public abstract class SearchIntegrationTest {
   /**
    * {
    * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp",
"latitude", "score", "is_alert"],
-   * "indices": ["bro", "snort", "metaalert"],
+   * "indices": ["bro", "snort"],
    * "query": "*",
    * "from": 0,
    * "size": 10,
@@ -346,7 +329,7 @@ public abstract class SearchIntegrationTest {
   /**
    * {
    * "fields": ["guid"],
-   * "indices": ["metaalert"],
+   * "indices": ["bro"],
    * "query": "*",
    * "from": 0,
    * "size": 10,
@@ -360,7 +343,7 @@ public abstract class SearchIntegrationTest {
    * }
    */
   @Multiline
-  public static String metaAlertsFieldQuery;
+  public static String sortByGuidQuery;
 
   /**
    * {
@@ -373,7 +356,7 @@ public abstract class SearchIntegrationTest {
    *   }
    * ],
    * "scoreField":"score",
-   * "indices": ["bro", "snort", "metaalert"],
+   * "indices": ["bro", "snort"],
    * "query": "*"
    * }
    */
@@ -398,7 +381,7 @@ public abstract class SearchIntegrationTest {
    *     }
    *   }
    * ],
-   * "indices": ["bro", "snort", "metaalert"],
+   * "indices": ["bro", "snort"],
    * "query": "*"
    * }
    */
@@ -453,6 +436,23 @@ public abstract class SearchIntegrationTest {
   public ExpectedException thrown = ExpectedException.none();
 
   @Test
+  public void all_query_returns_all_results() throws Exception {
+    SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
+    SearchResponse response = dao.search(request);
+    Assert.assertEquals(10, response.getTotal());
+    List<SearchResult> results = response.getResults();
+    Assert.assertEquals(10, results.size());
+    for(int i = 0;i < 5;++i) {
+      Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+      Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
+    }
+    for (int i = 5; i < 10; ++i) {
+      Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+      Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
+    }
+  }
+
+  @Test
   public void find_one_guid() throws Exception {
     GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
     Optional<Map<String, Object>> response = dao.getLatestResult(request);
@@ -463,240 +463,19 @@ public abstract class SearchIntegrationTest {
   }
 
   @Test
-  public void all_query_returns_all_results() throws Exception {
-    //All Query Testcase
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(10, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      Assert.assertEquals(10, results.size());
-      for(int i = 0;i < 5;++i) {
-        Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
-        Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
-      }
-      for (int i = 5; i < 10; ++i) {
-        Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
-        Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
-      }
-    }
-    //Get All Latest Guid Testcase
-    {
-      List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>()
{
-      });
-      Map<String, Document> docs = new HashMap<>();
-
-      for(Document doc : dao.getAllLatest(request)) {
-        docs.put(doc.getGuid(), doc);
-      }
-      Assert.assertEquals(2, docs.size());
-      Assert.assertTrue(docs.keySet().contains("bro-1"));
-      Assert.assertTrue(docs.keySet().contains("bro-2"));
-      for(Map.Entry<String, Document> kv : docs.entrySet()) {
-        Document d = kv.getValue();
-        Assert.assertEquals("bro", d.getDocument().get("source:type"));
-      }
-    }
-    //Filter test case
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(3, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
-      Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
-      Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
-      Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
-      Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
-      Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
-    }
-    //Sort test case
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(10, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      for(int i = 8001;i < 8011;++i) {
-        Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port"));
-      }
-    }
-    //Sort descending with missing fields
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(10, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      Assert.assertEquals(10, results.size());
-
-      // validate sorted order - there are only 2 with a 'threat:triage:score'
-      Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
-      Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
-
-      // the remaining are missing the 'threat:triage:score' and should be sorted last
-      Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(8).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(9).getSource().containsKey("threat:triage:score"));
-    }
-    //Sort ascending with missing fields
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(10, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      Assert.assertEquals(10, results.size());
-
-      // the remaining are missing the 'threat:triage:score' and should be sorted last
-      Assert.assertFalse(results.get(0).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(1).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
-      Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
-
-      // validate sorted order - there are only 2 with a 'threat:triage:score'
-      Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
-      Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
-    }
-    //pagination test case
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(10, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      Assert.assertEquals(3, results.size());
-      Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
-      Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
-      Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
-      Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
-      Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
-      Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
-    }
-    //Index query
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(5, response.getTotal());
-      List<SearchResult> results = response.getResults();
-      for(int i = 5,j=0;i > 0;i--,j++) {
-        Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
-        Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
-      }
-    }
-    //Facet query including all field types
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertEquals(12, response.getTotal());
-
-      Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
-      Assert.assertEquals(8, facetCounts.size());
-
-      // source:type
-      Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
-      Assert.assertEquals(2, sourceTypeCounts.size());
-      Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
-      Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
-
-      // ip_src_addr
-      Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
-      Assert.assertEquals(8, ipSrcAddrCounts.size());
-      Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
-      Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
-
-      // ip_src_port
-      Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
-      Assert.assertEquals(10, ipSrcPortCounts.size());
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
-      Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
-
-      // long_field
-      Map<String, Long> longFieldCounts = facetCounts.get("long_field");
-      Assert.assertEquals(2, longFieldCounts.size());
-      Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
-      Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
-
-      // timestamp
-      Map<String, Long> timestampCounts = facetCounts.get("timestamp");
-      Assert.assertEquals(10, timestampCounts.size());
-      Assert.assertEquals(new Long(1), timestampCounts.get("1"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("2"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("3"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("4"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("5"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("6"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("7"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("8"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("9"));
-      Assert.assertEquals(new Long(1), timestampCounts.get("10"));
-
-      // latitude
-      Map<String, Long> latitudeCounts = facetCounts.get("latitude");
-      Assert.assertEquals(2, latitudeCounts.size());
-      List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
-      Collections.sort(latitudeKeys);
-      Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
-      Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
-      Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
-      Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
-
-      // score
-      Map<String, Long> scoreFieldCounts = facetCounts.get("score");
-      Assert.assertEquals(4, scoreFieldCounts.size());
-      List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
-      Collections.sort(scoreFieldKeys);
-      Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
-      Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
-      Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
-      Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
-      Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
-      Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
-      Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
-      Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
-
-      // is_alert
-      Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
-      Assert.assertEquals(2, isAlertCounts.size());
-      Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
-      Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
-    }
-    //Bad facet query
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
-      try {
-        dao.search(request);
-        Assert.fail("Exception expected, but did not come.");
-      }
-      catch(InvalidSearchException ise) {
-        // success
-      }
-    }
-    //Disabled facet query
-    {
-      SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
-      SearchResponse response = dao.search(request);
-      Assert.assertNull(response.getFacetCounts());
+  public void get_all_latest_guid() throws Exception {
+    List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>()
{
+    });
+    Map<String, Document> docs = new HashMap<>();
+
+    for(Document doc : dao.getAllLatest(request)) {
+      docs.put(doc.getGuid(), doc);
     }
+    Assert.assertEquals(2, docs.size());
+    Assert.assertTrue(docs.keySet().contains("bro_1"));
+    Assert.assertTrue(docs.keySet().contains("snort_2"));
+    Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
+    Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
   }
 
   @Test
@@ -725,6 +504,42 @@ public abstract class SearchIntegrationTest {
   }
 
   @Test
+  public void sort_ascending_with_missing_fields() throws Exception {
+    SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
+    SearchResponse response = dao.search(request);
+    Assert.assertEquals(10, response.getTotal());
+    List<SearchResult> results = response.getResults();
+    Assert.assertEquals(10, results.size());
+
+    // the remaining are missing the 'threat:triage:score' and should be sorted last
+    for (int i = 0; i < 8; i++) {
+      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
+    }
+
+    // validate sorted order - there are only 2 with a 'threat:triage:score'
+    Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
+    Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
+  }
+
+  @Test
+  public void sort_descending_with_missing_fields() throws Exception {
+    SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
+    SearchResponse response = dao.search(request);
+    Assert.assertEquals(10, response.getTotal());
+    List<SearchResult> results = response.getResults();
+    Assert.assertEquals(10, results.size());
+
+    // validate sorted order - there are only 2 with a 'threat:triage:score'
+    Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
+    Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
+
+    // the remaining are missing the 'threat:triage:score' and should be sorted last
+    for (int i = 2; i < 10; i++) {
+      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
+    }
+  }
+
+  @Test
   public void results_are_paginated() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
@@ -755,7 +570,7 @@ public abstract class SearchIntegrationTest {
   public void facet_query_yields_field_types() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
-    Assert.assertEquals(12, response.getTotal());
+    Assert.assertEquals(10, response.getTotal());
     Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
     Assert.assertEquals(8, facetCounts.size());
     Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
@@ -1110,15 +925,15 @@ public abstract class SearchIntegrationTest {
   }
 
   @Test
-  public void searches_metaalerts_fields() throws Exception {
-    SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class);
+  public void sort_by_guid() throws Exception {
+    SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
-    Assert.assertEquals(2, response.getTotal());
+    Assert.assertEquals(5, response.getTotal());
     List<SearchResult> results = response.getResults();
-    for (int i = 0; i < 2; ++i) {
+    for (int i = 0; i < 5; ++i) {
       Map<String, Object> source = results.get(i).getSource();
       Assert.assertEquals(1, source.size());
-      Assert.assertEquals(source.get("guid"), "meta_" + (i + 1));
+      Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
     }
   }
 


Mime
View raw message