metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [23/53] [abbrv] metron git commit: METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005
Date Tue, 26 Jun 2018 21:15:39 GMT
METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/832296f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/832296f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/832296f0

Branch: refs/heads/master
Commit: 832296f0d4615bf95aed1c0742362c6127bd733a
Parents: d0a4e4c
Author: justinleet <justinjleet@gmail.com>
Authored: Wed May 2 10:10:27 2018 -0400
Committer: leet <leet@apache.org>
Committed: Wed May 2 10:10:27 2018 -0400

----------------------------------------------------------------------
 .../ElasticsearchSearchIntegrationTest.java     |   22 +-
 .../indexing/dao/SearchIntegrationTest.java     |   55 +-
 .../metron/solr/dao/SolrColumnMetadataDao.java  |   33 +-
 .../apache/metron/solr/dao/SolrSearchDao.java   |    3 +-
 .../integration/SolrSearchIntegrationTest.java  |  138 +-
 .../resources/config/bro/conf/managed-schema    |   50 -
 .../resources/config/bro/conf/solrconfig.xml    | 1601 ------------------
 .../resources/config/snort/conf/managed-schema  |   51 -
 .../resources/config/snort/conf/solrconfig.xml  | 1601 ------------------
 9 files changed, 178 insertions(+), 3376 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 5569c54..bb28abb 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
 import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
@@ -109,7 +110,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
    *        "type": "text",
    *        "fielddata" : "true"
    *     },
-   *     "duplicate_name_field": {
+   *     "ttl": {
    *        "type": "text",
    *        "fielddata" : "true"
    *     },
@@ -162,7 +163,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
    *        "snort_field": {
    *          "type": "integer"
    *        },
-   *        "duplicate_name_field": {
+   *        "ttl": {
    *          "type": "integer"
    *        },
    *        "alert": {
@@ -275,7 +276,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
       Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
       Assert.assertEquals(13, fieldTypes.size());
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
       Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
       Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
@@ -287,7 +288,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
       Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
     }
     // getColumnMetadata with only snort
@@ -295,7 +296,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
       Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
       Assert.assertEquals(14, fieldTypes.size());
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
-      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
       Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
       Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
@@ -306,7 +307,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
       Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
       Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
-      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
     }
   }
@@ -328,7 +329,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
     Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
     Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
     //NOTE: This is because the field is in both bro and snort and they have different types.
-    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
+    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
     Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
     Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
   }
@@ -349,6 +350,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest
{
     Assert.assertEquals(1, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals("bro", results.get(0).getSource().get("source:type"));
-    Assert.assertEquals("data 1", results.get(0).getSource().get("duplicate_name_field"));
+    Assert.assertEquals("data 1", results.get(0).getSource().get("ttl"));
+  }
+
+  @Override
+  protected String getSourceTypeField() {
+    return Constants.SENSOR_TYPE.replace('.', ':');
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 83046b8..56406f4 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -40,6 +40,7 @@ import org.apache.metron.integration.InMemoryComponent;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -47,11 +48,11 @@ import org.junit.rules.ExpectedException;
 public abstract class SearchIntegrationTest {
   /**
    * [
-   * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field":
10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"},
-   * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field":
20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point":
"48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"},
-   * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field":
10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point":
"50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"},
-   * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field":
10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"},
-   * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field":
10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"}
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field":
10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 1", "ttl": "data 1", "guid":"bro_1"},
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field":
20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point":
"48.5839,7.7455", "bro_field": "bro data 2", "ttl": "data 2", "guid":"bro_2"},
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field":
10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point":
"50.0,7.7455", "bro_field": "bro data 3", "ttl": "data 3", "guid":"bro_3"},
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field":
10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 4", "ttl": "data 4", "guid":"bro_4"},
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field":
10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point":
"48.5839,7.7455", "bro_field": "bro data 5", "ttl": "data 5", "guid":"bro_5"}
    * ]
    */
   @Multiline
@@ -59,11 +60,11 @@ public abstract class SearchIntegrationTest {
 
   /**
    * [
-   * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field":
10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point":
"50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1", "threat:triage:score":10.0},
-   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field":
10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2", "threat:triage:score":20.0},
-   * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field":
10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point":
"48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"},
-   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field":
20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point":
"48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"},
-   * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field":
10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point":
"48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"}
+   * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field":
10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point":
"50.0,7.7455", "snort_field": 10, "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0},
+   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field":
10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point":
"48.5839,7.7455", "snort_field": 20, "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0},
+   * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field":
10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point":
"48.5839,7.7455", "snort_field": 30, "ttl": 3, "guid":"snort_3"},
+   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field":
20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point":
"48.5839,7.7455", "snort_field": 40, "ttl": 4, "guid":"snort_4"},
+   * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field":
10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point":
"48.5839,7.7455", "snort_field": 50, "ttl": 5, "guid":"snort_5"}
    * ]
    */
   @Multiline
@@ -234,7 +235,7 @@ public abstract class SearchIntegrationTest {
    * }
    */
   @Multiline
-  public static String facetQuery;
+  public static String facetQueryRaw;
 
   /**
    * {
@@ -291,7 +292,7 @@ public abstract class SearchIntegrationTest {
 
   /**
    * {
-   * "facetFields": ["duplicate_name_field"],
+   * "facetFields": ["ttl"],
    * "indices": ["bro", "snort"],
    * "query": "*:*",
    * "from": 0,
@@ -457,7 +458,7 @@ public abstract class SearchIntegrationTest {
   /**
    * {
    * "indices": ["bro", "snort"],
-   * "query": "duplicate_name_field:\"data 1\"",
+   * "query": "ttl:\"data 1\"",
    * "from": 0,
    * "size": 10,
    * "sort": [
@@ -494,11 +495,11 @@ public abstract class SearchIntegrationTest {
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(10, results.size());
     for(int i = 0;i < 5;++i) {
-      Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+      Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField()));
       Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
     }
     for (int i = 5; i < 10; ++i) {
-      Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+      Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField()));
       Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
     }
   }
@@ -509,7 +510,7 @@ public abstract class SearchIntegrationTest {
     Optional<Map<String, Object>> response = dao.getLatestResult(request);
     Assert.assertTrue(response.isPresent());
     Map<String, Object> doc = response.get();
-    Assert.assertEquals("bro", doc.get("source:type"));
+    Assert.assertEquals("bro", doc.get(getSourceTypeField()));
     Assert.assertEquals("3", doc.get("timestamp").toString());
   }
 
@@ -524,8 +525,8 @@ public abstract class SearchIntegrationTest {
     Assert.assertEquals(2, docs.size());
     Assert.assertTrue(docs.keySet().contains("bro_1"));
     Assert.assertTrue(docs.keySet().contains("snort_2"));
-    Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
-    Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
+    Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField()));
+    Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField()));
   }
 
   @Test
@@ -534,11 +535,11 @@ public abstract class SearchIntegrationTest {
     SearchResponse response = dao.search(request);
     Assert.assertEquals(3, response.getTotal());
     List<SearchResult> results = response.getResults();
-    Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+    Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
     Assert.assertEquals("9", results.get(0).getSource().get("timestamp").toString());
-    Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
+    Assert.assertEquals("snort", results.get(1).getSource().get(getSourceTypeField()));
     Assert.assertEquals("7", results.get(1).getSource().get("timestamp").toString());
-    Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+    Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
     Assert.assertEquals("1", results.get(2).getSource().get("timestamp").toString());
   }
 
@@ -596,11 +597,11 @@ public abstract class SearchIntegrationTest {
     Assert.assertEquals(10, response.getTotal());
     List<SearchResult> results = response.getResults();
     Assert.assertEquals(3, results.size());
-    Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+    Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
     Assert.assertEquals("6", results.get(0).getSource().get("timestamp").toString());
-    Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
+    Assert.assertEquals("bro", results.get(1).getSource().get(getSourceTypeField()));
     Assert.assertEquals("5", results.get(1).getSource().get("timestamp").toString());
-    Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+    Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
     Assert.assertEquals("4", results.get(2).getSource().get("timestamp").toString());
   }
 
@@ -611,19 +612,20 @@ public abstract class SearchIntegrationTest {
     Assert.assertEquals(5, response.getTotal());
     List<SearchResult> results = response.getResults();
     for (int i = 5, j = 0; i > 0; i--, j++) {
-      Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
+      Assert.assertEquals("bro", results.get(j).getSource().get(getSourceTypeField()));
       Assert.assertEquals(i + "", results.get(j).getSource().get("timestamp").toString());
     }
   }
 
   @Test
   public void facet_query_yields_field_types() throws Exception {
+    String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField());
     SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
     Assert.assertEquals(10, response.getTotal());
     Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
     Assert.assertEquals(8, facetCounts.size());
-    Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
+    Map<String, Long> sourceTypeCounts = facetCounts.get(getSourceTypeField());
     Assert.assertEquals(2, sourceTypeCounts.size());
     Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
     Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
@@ -951,4 +953,5 @@ public abstract class SearchIntegrationTest {
   protected abstract IndexDao createDao() throws Exception;
   protected abstract InMemoryComponent startIndex() throws Exception;
   protected abstract void loadTestData() throws Exception;
+  protected abstract String getSourceTypeField();
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
index 61f0209..1e074a9 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
@@ -20,17 +20,25 @@ package org.apache.metron.solr.dao;
 import com.google.common.collect.Sets;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import org.apache.metron.indexing.dao.ColumnMetadataDao;
 import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.LukeRequest;
 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest.DynamicFields;
+import org.apache.solr.client.solrj.response.LukeResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse.DynamicFieldsResponse;
 import org.apache.solr.common.SolrException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +57,7 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao {
     fieldTypeMap.put("pfloat", FieldType.FLOAT);
     fieldTypeMap.put("pdouble", FieldType.DOUBLE);
     fieldTypeMap.put("boolean", FieldType.BOOLEAN);
+    fieldTypeMap.put("ip", FieldType.IP);
     solrTypeMap = Collections.unmodifiableMap(fieldTypeMap);
   }
 
@@ -104,12 +113,30 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao {
     return indexColumnMetadata;
   }
 
-  protected List<Map<String, Object>> getIndexFields(String index) throws IOException,
SolrServerException {
+  protected List<Map<String, Object>> getIndexFields(String index)
+      throws IOException, SolrServerException {
     CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();
     client.setDefaultCollection(index);
+
+    List<Map<String, Object>> indexFields = new ArrayList<>();
+
+    // Get all the fields in use, including dynamic fields
+    LukeRequest lukeRequest = new LukeRequest();
+    LukeResponse lukeResponse = lukeRequest.process(client);
+    for (Entry<String, LukeResponse.FieldInfo> field : lukeResponse.getFieldInfo().entrySet())
{
+      Map<String, Object> fieldData = new HashMap<>();
+      fieldData.put("name", field.getValue().getName());
+      fieldData.put("type", field.getValue().getType());
+      indexFields.add(fieldData);
+
+    }
+
+    // Get all the schema fields
     SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client)
-            .getSchemaRepresentation();
-    return schemaRepresentation.getFields();
+        .getSchemaRepresentation();
+    indexFields.addAll(schemaRepresentation.getFields());
+
+    return indexFields;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 272b96a..f1b7102 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -47,6 +47,7 @@ import org.apache.solr.client.solrj.response.PivotField;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -91,7 +92,7 @@ public class SolrSearchDao implements SearchDao {
       SolrQuery query = buildSearchRequest(searchRequest);
       QueryResponse response = client.query(query);
       return buildSearchResponse(searchRequest, response);
-    } catch (IOException | SolrServerException e) {
+    } catch (SolrException | IOException | SolrServerException e) {
       String msg = e.getMessage();
       LOG.error(msg, e);
       throw new InvalidSearchException(msg, e);

http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index a9ce650..92941cd 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -21,6 +21,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import org.apache.metron.common.Constants;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.indexing.dao.AccessConfig;
 import org.apache.metron.indexing.dao.IndexDao;
@@ -36,12 +37,16 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.json.simple.JSONArray;
 import org.json.simple.parser.JSONParser;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class SolrSearchIntegrationTest extends SearchIntegrationTest {
 
   private SolrComponent solrComponent;
 
+  private String broData = SearchIntegrationTest.broData.replace("source:type", "source.type");
+  private String snortData = SearchIntegrationTest.snortData.replace("source:type", "source.type");
+
   @Override
   protected IndexDao createDao() throws Exception {
     AccessConfig config = new AccessConfig();
@@ -60,9 +65,10 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
 
   @Override
   protected InMemoryComponent startIndex() throws Exception {
+
     solrComponent = new SolrComponent.Builder()
-        .addCollection("bro", "../metron-solr/src/test/resources/config/bro/conf")
-        .addCollection("snort", "../metron-solr/src/test/resources/config/snort/conf")
+        .addCollection("bro", "../metron-solr/src/main/config/schema/bro")
+        .addCollection("snort", "../metron-solr/src/main/config/schema/snort")
         .build();
     solrComponent.start();
     return solrComponent;
@@ -83,65 +89,122 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest
{
   }
 
   @Override
+  @Test
   public void returns_column_metadata_for_specified_indices() throws Exception {
     // getColumnMetadata with only bro
     {
       Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
-      Assert.assertEquals(12, fieldTypes.size());
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+      // Don't test all 256, just test a sample of different fields
+      Assert.assertEquals(261, fieldTypes.size());
+
+      // Fields present in both with same type
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
-      Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
-      Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
-      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
-      Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
       Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+      // Bro only field
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));
+
+      // A dynamic field present in both with same type
+      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+      // Dyanamic field present in both with nonstandard types.
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+
+      // Field with nonstandard type
+      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+      // Bro only field in the dynamic catch all
+      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+
+      // A field is in both bro and snort and they have different types.
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
+
+      // Field only present in Snort
+      Assert.assertEquals(null, fieldTypes.get("dgmlen"));
+
+      // Field that doesn't exist
+      Assert.assertEquals(null, fieldTypes.get("fake.field"));
     }
     // getColumnMetadata with only snort
     {
       Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
-      Assert.assertEquals(13, fieldTypes.size());
-      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
-      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+      Assert.assertEquals(31, fieldTypes.size());
+
+      // Fields present in both with same type
       Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
-      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
       Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
-      Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
-      Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
-      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
-      Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
       Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+      // Snort only field
+      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));
+
+      // A dynamic field present in both with same type
+      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+      // Dyanamic field present in both with nonstandard types.
       Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
-      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+
+      // Field with nonstandard type
+      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+      // Snort only field in the dynamic catch all
+      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+
+      // A field is in both bro and snort and they have different types.
+      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
+
+      // Field only present in Bro
+      Assert.assertEquals(null, fieldTypes.get("username"));
+
+      // Field that doesn't exist
+      Assert.assertEquals(null, fieldTypes.get("fake.field"));
     }
   }
 
   @Override
+  @Test
   public void returns_column_data_for_multiple_indices() throws Exception {
     Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro",
"snort"));
-    Assert.assertEquals(14, fieldTypes.size());
+    // Don't test everything, just test a variety of fields, including fields across collections.
+
+    // Fields present in both with same type
     Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
-    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
-    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+    Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
     Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
-    Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
-    Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
-    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
-    Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
     Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+    // Bro only field
+    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));
+
+    // Snort only field
+    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));
+
+    // A dynamic field present in both with same type
+    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+    // Dyanamic field present in both with nonstandard types.
     Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
-    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
-    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
-    //NOTE: This is because the field is in both bro and snort and they have different types.
-    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
-    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
+
+    // Field present in both with nonstandard type
+    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+    // Bro only field in the dynamic catch all
+    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+
+    // Snort only field in the dynamic catch all
+    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+
+    // A field is in both bro and snort and they have different types.
+    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
+
+    // Field that doesn't exist
+    Assert.assertEquals(null, fieldTypes.get("fake.field"));
   }
 
   @Test
@@ -150,4 +213,9 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
     SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
     SearchResponse response = dao.search(request);
   }
+
+  @Override
+  protected String getSourceTypeField() {
+    return Constants.SENSOR_TYPE;
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
deleted file mode 100644
index 3bcb2bc..0000000
--- a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<schema name="example" version="1.6">
-   <field name="_version_" type="plong" indexed="false" stored="false"/>
-   <field name="_root_" type="string" indexed="true" stored="false" docValues="false"
/>
-
-   <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false"
/>
-
-   <field name="source:type" type="string" indexed="true" stored="true"/>
-   <field name="ip_src_addr" type="string" indexed="true" stored="true" />
-   <field name="ip_src_port" type="pint" indexed="true" stored="true"/>
-   <field name="long_field" type="plong" indexed="true" stored="true" />
-   <field name="timestamp" type="plong" indexed="true" stored="true" />
-   <field name="latitude" type="pfloat" indexed="true" stored="true" />
-   <field name="score" type="pdouble" indexed="true" stored="true" />
-   <field name="is_alert" type="boolean" indexed="true" stored="true"/>
-   <field name="location_point"  type="location" indexed="true" stored="true"/>
-   <field name="bro_field" type="string" indexed="true" stored="true" />
-   <field name="duplicate_name_field" type="string" indexed="true" stored="true" />
-
-   <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
-
- <uniqueKey>guid</uniqueKey>
-
-
-    <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
-    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
-    <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
-    <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
-    <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
-    <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
-    <fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true"/>
-    <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField"
/>
-
-</schema>


Mime
View raw message