metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject metron git commit: METRON-1226 Searching Can Errantly Query the Wrong Indices (nickwallen) closes apache/metron#793
Date Mon, 16 Oct 2017 16:21:10 GMT
Repository: metron
Updated Branches:
  refs/heads/master 4ee55702e -> 67cba8130


METRON-1226 Searching Can Errantly Query the Wrong Indices (nickwallen) closes apache/metron#793


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/67cba813
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/67cba813
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/67cba813

Branch: refs/heads/master
Commit: 67cba8130af3a804e1aa5ccf6ea64c9e9e1a07bf
Parents: 4ee5570
Author: nickwallen <nick@nickallen.org>
Authored: Mon Oct 16 12:20:01 2017 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Mon Oct 16 12:20:01 2017 -0400

----------------------------------------------------------------------
 .../CURRENT/package/scripts/indexing_master.py  |  2 +-
 .../elasticsearch/dao/ElasticsearchDao.java     | 76 +++++++++++++-------
 .../elasticsearch/utils/ElasticsearchUtils.java | 49 ++++++++++++-
 .../writer/ElasticsearchWriter.java             | 47 ++++--------
 .../elasticsearch/dao/ElasticsearchDaoTest.java | 38 +++++++---
 .../ElasticsearchSearchIntegrationTest.java     |  2 +-
 .../matcher/SearchRequestMatcher.java           | 43 +++++++++--
 .../metron/indexing/dao/MetaAlertDao.java       |  2 +-
 .../indexing/dao/InMemoryMetaAlertDao.java      |  2 +-
 .../indexing/dao/SearchIntegrationTest.java     |  2 +-
 10 files changed, 180 insertions(+), 83 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 371cab0..8f156d6 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -145,7 +145,7 @@ class Indexing(Script):
             'curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}')
         Execute(error_cmd, logoutput=True)
         error_cmd = ambari_format(
-            'curl -s -XPOST http://{es_http_url}/metaalerts -d @{meta_index_path}')
+            'curl -s -XPOST http://{es_http_url}/metaalert_index -d @{meta_index_path}')
         Execute(error_cmd, logoutput=True)
 
     def elasticsearch_template_delete(self, env):

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index aa56ed0..f2f1b38 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -19,19 +19,6 @@ package org.apache.metron.elasticsearch.dao;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
 import org.apache.metron.indexing.dao.AccessConfig;
 import org.apache.metron.indexing.dao.IndexDao;
@@ -76,6 +63,23 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.INDEX_NAME_DELIMITER;
+
+
 public class ElasticsearchDao implements IndexDao {
 
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -131,9 +135,9 @@ public class ElasticsearchDao implements IndexDao {
             .size(searchRequest.getSize())
             .from(searchRequest.getFrom())
             .query(queryBuilder)
-
             .trackScores(true);
-    searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(),
getElasticsearchSortOrder(sortField.getSortOrder())));Optional<List<String>> fields
= searchRequest.getFields();
+    searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(),
getElasticsearchSortOrder(sortField.getSortOrder())));
+    Optional<List<String>> fields = searchRequest.getFields();
     if (fields.isPresent()) {
       searchSourceBuilder.fields(fields.get());
     } else {
@@ -143,7 +147,7 @@ public class ElasticsearchDao implements IndexDao {
     if (facetFields.isPresent()) {
       facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field)));
     }
-    String[] wildcardIndices = searchRequest.getIndices().stream().map(index -> String.format("%s*",
index)).toArray(value -> new String[searchRequest.getIndices().size()]);
+    String[] wildcardIndices = wildcardIndices(searchRequest.getIndices());
     org.elasticsearch.action.search.SearchResponse elasticsearchResponse;
     try {
       elasticsearchResponse = client.search(new org.elasticsearch.action.search.SearchRequest(wildcardIndices)
@@ -179,11 +183,13 @@ public class ElasticsearchDao implements IndexDao {
     final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
     searchSourceBuilder.query(new QueryStringQueryBuilder(groupRequest.getQuery()));
     searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest, 0));
-    String[] wildcardIndices = groupRequest.getIndices().stream().map(index -> String.format("%s*",
index)).toArray(value -> new String[groupRequest.getIndices().size()]);
-    org.elasticsearch.action.search.SearchResponse elasticsearchResponse;
+    String[] wildcardIndices = wildcardIndices(groupRequest.getIndices());
+    org.elasticsearch.action.search.SearchRequest request;
+    org.elasticsearch.action.search.SearchResponse response;
+
     try {
-      elasticsearchResponse = client.search(new org.elasticsearch.action.search.SearchRequest(wildcardIndices)
-          .source(searchSourceBuilder)).actionGet();
+      request = new org.elasticsearch.action.search.SearchRequest(wildcardIndices).source(searchSourceBuilder);
+      response = client.search(request).actionGet();
     } catch (SearchPhaseExecutionException e) {
       throw new InvalidSearchException("Could not execute search", e);
     }
@@ -195,10 +201,17 @@ public class ElasticsearchDao implements IndexDao {
     }
     GroupResponse groupResponse = new GroupResponse();
     groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField());
-    groupResponse.setGroupResults(getGroupResults(groupRequest, 0, elasticsearchResponse.getAggregations(),
commonColumnMetadata));
+    groupResponse.setGroupResults(getGroupResults(groupRequest, 0, response.getAggregations(),
commonColumnMetadata));
     return groupResponse;
   }
 
+  private String[] wildcardIndices(List<String> indices) {
+    return indices
+            .stream()
+            .map(index -> String.format("%s%s*", index, INDEX_NAME_DELIMITER))
+            .toArray(value -> new String[indices.size()]);
+  }
+
   @Override
   public synchronized void init(AccessConfig config) {
     if(this.client == null) {
@@ -301,11 +314,18 @@ public class ElasticsearchDao implements IndexDao {
   @Override
   public Map<String, Map<String, FieldType>> getColumnMetadata(List<String>
indices) throws IOException {
     Map<String, Map<String, FieldType>> allColumnMetadata = new HashMap<>();
-    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings
=
-            client.admin().indices().getMappings(new GetMappingsRequest().indices(getLatestIndices(indices))).actionGet().getMappings();
-    for(Object index: mappings.keys().toArray()) {
+    String[] latestIndices = getLatestIndices(indices);
+    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings
= client
+            .admin()
+            .indices()
+            .getMappings(new GetMappingsRequest().indices(latestIndices))
+            .actionGet()
+            .getMappings();
+    for(Object key: mappings.keys().toArray()) {
+      String indexName = key.toString();
+
       Map<String, FieldType> indexColumnMetadata = new HashMap<>();
-      ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(index.toString());
+      ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(indexName);
       Iterator<String> mappingIterator = mapping.keysIt();
       while(mappingIterator.hasNext()) {
         MappingMetaData mappingMetaData = mapping.get(mappingIterator.next());
@@ -314,7 +334,9 @@ public class ElasticsearchDao implements IndexDao {
           indexColumnMetadata.put(field, elasticsearchSearchTypeMap.getOrDefault(map.get(field).get("type"),
FieldType.OTHER));
         }
       }
-      allColumnMetadata.put(index.toString().split("_index_")[0], indexColumnMetadata);
+
+      String baseIndexName = ElasticsearchUtils.getBaseIndexName(indexName);
+      allColumnMetadata.put(baseIndexName, indexColumnMetadata);
     }
     return allColumnMetadata;
   }
@@ -348,7 +370,7 @@ public class ElasticsearchDao implements IndexDao {
     String[] indices = client.admin().indices().prepareGetIndex().setFeatures().get().getIndices();
     for (String index : indices) {
       if (!ignoredIndices.contains(index)) {
-        int prefixEnd = index.indexOf("_index_");
+        int prefixEnd = index.indexOf(INDEX_NAME_DELIMITER);
         if (prefixEnd != -1) {
           String prefix = index.substring(0, prefixEnd);
           if (includeIndices.contains(prefix)) {

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index c7c4d90..4c9933b 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -20,8 +20,8 @@ package org.apache.metron.elasticsearch.utils;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import org.apache.commons.lang.StringUtils;
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.elasticsearch.writer.ElasticsearchWriter;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -29,13 +29,30 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.lang.String.format;
 
 public class ElasticsearchUtils {
 
   private static ThreadLocal<Map<String, SimpleDateFormat>> DATE_FORMAT_CACHE
           = ThreadLocal.withInitial(() -> new HashMap<>());
 
+  /**
+   * A delimiter that is appended to the user-defined index name to separate
+   * the index's date postfix.
+   *
+   * For example, if the user-defined index name is 'bro', the delimiter is
+   * '_index', and the index's date postfix is '2017.10.03.19', then the actual
+   * index name should be 'bro_index_2017.10.03.19'.
+   */
+  public static final String INDEX_NAME_DELIMITER = "_index";
+
   public static SimpleDateFormat getIndexFormat(WriterConfiguration configurations) {
     return getIndexFormat(configurations.getGlobalConfig());
   }
@@ -45,15 +62,41 @@ public class ElasticsearchUtils {
     return DATE_FORMAT_CACHE.get().computeIfAbsent(format, SimpleDateFormat::new);
   }
 
+  /**
+   * Builds the name of an Elasticsearch index.
+   * @param sensorType The sensor type; bro, yaf, snort, ...
+   * @param indexPostfix The index postfix; most often a formatted date.
+   * @param configurations User-defined configuration for the writers.
+   */
   public static String getIndexName(String sensorType, String indexPostfix, WriterConfiguration
configurations) {
     String indexName = sensorType;
     if (configurations != null) {
       indexName = configurations.getIndex(sensorType);
     }
-    indexName = indexName + "_index_" + indexPostfix;
+    indexName = indexName + INDEX_NAME_DELIMITER + "_" + indexPostfix;
     return indexName;
   }
 
+  /**
+   * Extracts the base index name from a full index name.
+   *
+   * For example, given an index named 'bro_index_2017.01.01.01', the base
+   * index name is 'bro'.
+   *
+   * @param indexName The full index name including delimiter and date postfix.
+   * @return The base index name.
+   */
+  public static String getBaseIndexName(String indexName) {
+
+    String[] parts = indexName.split(INDEX_NAME_DELIMITER);
+    if(parts.length < 1 || StringUtils.isEmpty(parts[0])) {
+      String msg = format("Unexpected index name; index=%s, delimiter=%s", indexName, INDEX_NAME_DELIMITER);
+      throw new IllegalStateException(msg);
+    }
+
+    return parts[0];
+  }
+
   public static TransportClient getClient(Map<String, Object> globalConfiguration,
Map<String, String> optionalSettings) {
     Settings.Builder settingsBuilder = Settings.settingsBuilder();
     settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername"));

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index dd32532..bc9eccc 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -18,18 +18,6 @@
 package org.apache.metron.elasticsearch.writer;
 
 import org.apache.metron.common.Constants;
-import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import java.io.Serializable;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.common.interfaces.FieldNameConverter;
 import org.apache.metron.common.writer.BulkMessageWriter;
@@ -46,13 +34,19 @@ import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Serializable
{
 
   private Map<String, String> optionalSettings;
   private transient TransportClient client;
   private SimpleDateFormat dateFormat;
-  private static final Logger LOG = LoggerFactory
-          .getLogger(ElasticsearchWriter.class);
+  private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchWriter.class);
   private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
 
   public ElasticsearchWriter withOptionalSettings(Map<String, String> optionalSettings)
{
@@ -64,34 +58,24 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>,
Seria
   public void init(Map stormConf, TopologyContext topologyContext, WriterConfiguration configurations)
{
     Map<String, Object> globalConfiguration = configurations.getGlobalConfig();
     client = ElasticsearchUtils.getClient(globalConfiguration, optionalSettings);
-    dateFormat = new SimpleDateFormat((String) globalConfiguration.get("es.date.format"));
+    dateFormat = ElasticsearchUtils.getIndexFormat(globalConfiguration);
   }
 
+
   @Override
   public BulkWriterResponse write(String sensorType, WriterConfiguration configurations,
Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
-    String indexPostfix = dateFormat.format(new Date());
+    final String indexPostfix = dateFormat.format(new Date());
     BulkRequestBuilder bulkRequest = client.prepareBulk();
 
     for(JSONObject message: messages) {
 
-      String indexName = sensorType;
-
-      if (configurations != null) {
-        indexName = configurations.getIndex(sensorType);
-      }
-
-      indexName = indexName + "_index_" + indexPostfix;
-
       JSONObject esDoc = new JSONObject();
       for(Object k : message.keySet()){
-
-        deDot(k.toString(),message,esDoc);
-
+        deDot(k.toString(), message, esDoc);
       }
 
-      IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName,
-              sensorType + "_doc");
-
+      String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
+      IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, sensorType
+ "_doc");
       indexRequestBuilder = indexRequestBuilder.setSource(esDoc.toJSONString());
       String guid = (String)esDoc.get(Constants.GUID);
       if(guid != null) {
@@ -102,12 +86,11 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>,
Seria
       if(ts != null) {
         indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
       }
-      bulkRequest.add(indexRequestBuilder);
 
+      bulkRequest.add(indexRequestBuilder);
     }
 
     BulkResponse bulkResponse = bulkRequest.execute().actionGet();
-
     return buildWriteReponse(tuples, bulkResponse);
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index c28ffc7..7c33018 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
@@ -27,9 +27,7 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 import org.mockito.Mock;
 
 import java.util.Arrays;
@@ -54,42 +52,53 @@ public class ElasticsearchDaoTest {
     AccessConfig config = mock(AccessConfig.class);
     when(config.getMaxSearchResults()).thenReturn(50);
     searchService = new ElasticsearchDao(client, config);
-
   }
 
   @Test
   public void searchShouldProperlyBuildSearchRequest() throws Exception {
+
+    // setup the mock client
     SearchHit searchHit1 = mock(SearchHit.class);
     when(searchHit1.getId()).thenReturn("id1");
     when(searchHit1.getSource()).thenReturn(new HashMap<String, Object>(){{ put("field",
"value1"); }});
     when(searchHit1.getScore()).thenReturn(0.1f);
+
     SearchHit searchHit2 = mock(SearchHit.class);
     when(searchHit2.getId()).thenReturn("id2");
     when(searchHit2.getSource()).thenReturn(new HashMap<String, Object>(){{ put("field",
"value2"); }});
     when(searchHit2.getScore()).thenReturn(0.2f);
+
     SearchHits searchHits = mock(SearchHits.class);
     when(searchHits.getHits()).thenReturn(new SearchHit[]{searchHit1, searchHit2});
     when(searchHits.getTotalHits()).thenReturn(2L);
+
     org.elasticsearch.action.search.SearchResponse elasticsearchResponse = mock(org.elasticsearch.action.search.SearchResponse.class);
     when(elasticsearchResponse.getHits()).thenReturn(searchHits);
+
     ActionFuture actionFuture = mock(ActionFuture.class);
     when(actionFuture.actionGet()).thenReturn(elasticsearchResponse);
     when(client.search(any())).thenReturn(actionFuture);
 
+    // "sort by" fields for the search request
+    SortField[] sortFields = {
+            sortBy("sortField1", SortOrder.DESC),
+            sortBy("sortField2", SortOrder.ASC)
+    };
+
+    // create a search request
     SearchRequest searchRequest = new SearchRequest();
     searchRequest.setSize(2);
     searchRequest.setIndices(Arrays.asList("bro", "snort"));
     searchRequest.setFrom(5);
-    SortField sortField1 = new SortField();
-    sortField1.setField("sortField1");
-    sortField1.setSortOrder(SortOrder.DESC.toString());
-    SortField sortField2 = new SortField();
-    sortField2.setField("sortField2");
-    sortField2.setSortOrder(SortOrder.ASC.toString());
-    searchRequest.setSort(Arrays.asList(sortField1, sortField2));
+    searchRequest.setSort(Arrays.asList(sortFields));
     searchRequest.setQuery("some query");
+
+    // submit the search request
     SearchResponse searchResponse = searchService.search(searchRequest);
-    verify(client, times(1)).search(argThat(new SearchRequestMatcher(new String[]{"bro*",
"snort*"}, "some query", 2, 5, new SortField[]{sortField1, sortField2})));
+
+    // validate
+    String[] expectedIndices = {"bro_index*", "snort_index*"};
+    verify(client).search(argThat(new SearchRequestMatcher(expectedIndices, "some query",
2, 5, sortFields)));
     assertEquals(2, searchResponse.getTotal());
     List<SearchResult> actualSearchResults = searchResponse.getResults();
     assertEquals(2, actualSearchResults.size());
@@ -102,6 +111,13 @@ public class ElasticsearchDaoTest {
     verifyNoMoreInteractions(client);
   }
 
+  private SortField sortBy(String field, SortOrder order) {
+    SortField sortField = new SortField();
+    sortField.setField(field);
+    sortField.setSortOrder(order.toString());
+    return sortField;
+  }
+
   @Test
   public void searchShouldThrowExceptionWhenMaxResultsAreExceeded() throws Exception {
     SearchRequest searchRequest = new SearchRequest();

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index e21bb13..e7b609e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -155,7 +155,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
     JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData);
     for(Object o: metaAlertArray) {
       JSONObject jsonObject = (JSONObject) o;
-      IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalerts",
"metaalert_doc");
+      IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalert_index",
"metaalert_doc");
       indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
       bulkRequest.add(indexRequestBuilder);
     }

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java
index 9d69471..417e48b 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/matcher/SearchRequestMatcher.java
@@ -25,17 +25,24 @@ import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.hamcrest.Description;
 import org.mockito.ArgumentMatcher;
 
 import java.util.Arrays;
 
 public class SearchRequestMatcher extends ArgumentMatcher<SearchRequest> {
 
-  private String[] expectedIndicies;
+  private String[] expectedIndices;
+  private String[] actualIndices;
+
   private BytesReference expectedSource;
+  private BytesReference actualSource;
+
+  private boolean indicesMatch;
+  private boolean sourcesMatch;
 
   public SearchRequestMatcher(String[] indices, String query, int size, int from, SortField[]
sortFields) {
-    expectedIndicies = indices;
+    expectedIndices = indices;
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
             .size(size)
             .from(from)
@@ -53,8 +60,34 @@ public class SearchRequestMatcher extends ArgumentMatcher<SearchRequest>
{
   @Override
   public boolean matches(Object o) {
     SearchRequest searchRequest = (SearchRequest) o;
-    boolean indiciesMatch = Arrays.equals(expectedIndicies, searchRequest.indices());
-    boolean sourcesMatch = searchRequest.source().equals(expectedSource);
-    return indiciesMatch && sourcesMatch;
+
+    actualIndices = searchRequest.indices();
+    actualSource = searchRequest.source();
+
+    indicesMatch = Arrays.equals(expectedIndices, actualIndices);
+    sourcesMatch = expectedSource.equals(actualSource);
+
+    return indicesMatch && sourcesMatch;
+  }
+
+  @Override
+  public void describeTo(Description description) {
+    if(!indicesMatch) {
+      description.appendText("Bad search request indices: ");
+      description.appendText(" expected=");
+      description.appendValue(expectedIndices);
+      description.appendText(", got=");
+      description.appendValue(actualIndices);
+      description.appendText("  ");
+    }
+
+    if(!sourcesMatch) {
+      description.appendText("Bad search request sources: ");
+      description.appendText(" expected=");
+      description.appendValue(expectedSource);
+      description.appendText(", got=");
+      description.appendValue(actualSource);
+      description.appendText("  ");
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
index 4e0851b..05746c4 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
@@ -27,7 +27,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
 
 public interface MetaAlertDao extends IndexDao {
 
-  String METAALERTS_INDEX = "metaalerts";
+  String METAALERTS_INDEX = "metaalert";
   String METAALERT_TYPE = "metaalert";
   String METAALERT_DOC = METAALERT_TYPE + "_doc";
   String THREAT_FIELD_DEFAULT = "threat:triage:score";

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
index 8807bbc..39c0001 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
@@ -52,7 +52,7 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
 
   /**
    * {
-   * "indices": ["metaalerts"],
+   * "indices": ["metaalert"],
    * "query": "alert|guid:${GUID}",
    * "from": 0,
    * "size": 10,

http://git-wip-us.apache.org/repos/asf/metron/blob/67cba813/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index e2a37f1..2961d96 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -279,7 +279,7 @@ public abstract class SearchIntegrationTest {
   /**
    * {
    * "fields": ["guid"],
-   * "indices": ["metaalerts"],
+   * "indices": ["metaalert"],
    * "query": "*",
    * "from": 0,
    * "size": 10,


Mime
View raw message