metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [18/53] [abbrv] metron git commit: METRON-1526 Location field types cause DocValuesField appear more than once error (merrimanr via justinleet) closes apache/metron#995
Date Tue, 26 Jun 2018 21:15:34 GMT
METRON-1526 Location field types cause DocValuesField appear more than once error (merrimanr
via justinleet) closes apache/metron#995


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8d7843e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8d7843e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8d7843e

Branch: refs/heads/master
Commit: f8d7843e93e410147257cca40d83b2c2d4963f1b
Parents: 7ef4b77
Author: merrimanr <merrimanr@gmail.com>
Authored: Thu Apr 26 09:01:03 2018 -0400
Committer: leet <leet@apache.org>
Committed: Thu Apr 26 09:01:03 2018 -0400

----------------------------------------------------------------------
 .../indexing/dao/UpdateIntegrationTest.java     | 21 +++++++++---
 metron-platform/metron-solr/README.md           | 16 +++++++++
 .../src/main/config/schema/bro/schema.xml       |  1 +
 .../src/main/config/schema/snort/schema.xml     |  1 +
 .../src/main/config/schema/yaf/schema.xml       |  1 +
 .../integration/SolrUpdateIntegrationTest.java  | 34 ++++++++++++++++++--
 .../integration/components/SolrComponent.java   |  7 +++-
 7 files changed, 74 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index 369fa79..471acf6 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -31,11 +31,14 @@ import org.apache.metron.hbase.mock.MockHTable;
 import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.indexing.dao.update.ReplaceRequest;
 import org.apache.metron.integration.InMemoryComponent;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+
 public abstract class UpdateIntegrationTest {
 
   private static final int MAX_RETRIES = 10;
@@ -46,6 +49,7 @@ public abstract class UpdateIntegrationTest {
   private static String index;
   private static MockHTable table;
   private static IndexDao hbaseDao;
+  private static AccessConfig accessConfig;
 
   protected static MultiIndexDao dao;
   protected static InMemoryComponent indexComponent;
@@ -62,20 +66,24 @@ public abstract class UpdateIntegrationTest {
       table = (MockHTable)tableProvider.getTable(config, TABLE_NAME);
 
       hbaseDao = new HBaseDao();
-      AccessConfig accessConfig = new AccessConfig();
+      accessConfig = new AccessConfig();
       accessConfig.setTableProvider(tableProvider);
       Map<String, Object> globalConfig = createGlobalConfig();
       globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME);
       globalConfig.put(HBaseDao.HBASE_CF, CF);
       accessConfig.setGlobalConfigSupplier(() -> globalConfig);
-
-      dao = new MultiIndexDao(hbaseDao, createDao());
-      dao.init(accessConfig);
     }
   }
 
+  protected AccessConfig getAccessConfig() {
+    return accessConfig;
+  }
+
   @Test
   public void test() throws Exception {
+    dao = new MultiIndexDao(hbaseDao, createDao());
+    dao.init(getAccessConfig());
+
     List<Map<String, Object>> inputData = new ArrayList<>();
     for(int i = 0; i < 10;++i) {
       final String name = "message" + i;
@@ -181,6 +189,11 @@ public abstract class UpdateIntegrationTest {
     }
   }
 
+  @After
+  public void reset() throws Exception {
+    indexComponent.reset();
+  }
+
   @AfterClass
   public static void teardown() {
     if(indexComponent != null) {

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index 4321114..0d525ec 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -95,3 +95,19 @@ As of now, we have mapped out the Schemas in `src/main/config/schema`.
 Ambari will eventually install these, but at the moment it's manual and
 you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here)
in general
 and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html)
if you'd like to know more about schemas in Solr.
+
+In Metron's Solr DAO implementation, document updates involve reading a document, applying
the update and replacing the original by reindexing the whole document.  
+Indexing LatLonType and PointType field types stores data in internal fields that should
not be returned in search results.  For these fields a dynamic field type matching the suffix
needs to be added to store the data points.
+Solr 6+ comes with a new LatLonPointSpatialField field type that should be used instead of
LatLonType if possible.  Otherwise, a LatLongType field should be defined as:
+```
+<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+<dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
+<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+```
+A PointType field should be defined as:
+```
+<dynamicField name="*.point" type="point" multiValued="false" docValues="false"/>
+<dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/>
+<fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/>
+```
+If any copy fields are defined, stored and docValues should be set to false.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
index 4ce8d6b..b463366 100644
--- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
@@ -659,6 +659,7 @@
 
   <!-- Geo Enrichment Fields -->
   <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+  <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
   <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
index ae13b8d..129c0f0 100644
--- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -52,6 +52,7 @@
 
   <!-- Geo Enrichment Fields -->
   <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+  <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
   <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
index edccd9d..f3abb14 100644
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -58,6 +58,7 @@
 
   <!-- Geo Enrichment Fields -->
   <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+  <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
   <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
   <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index faa4ec4..232c727 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -20,15 +20,21 @@ package org.apache.metron.solr.integration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+
 import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
 import org.apache.metron.indexing.dao.UpdateIntegrationTest;
+import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.integration.InMemoryComponent;
 import org.apache.metron.solr.dao.SolrDao;
 import org.apache.metron.solr.integration.components.SolrComponent;
+import org.junit.Assert;
+import org.junit.Test;
 
 public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
 
-  SolrComponent solrComponent;
+  protected static SolrComponent solrComponent;
 
   @Override
   protected String getIndexName() {
@@ -49,7 +55,7 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
 
   @Override
   protected InMemoryComponent startIndex() throws Exception {
-    solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/test/resources/config/test/conf").build();
+    solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro").build();
     solrComponent.start();
     return solrComponent;
   }
@@ -70,4 +76,28 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
       throws Exception {
     return solrComponent.getAllIndexedDocs(indexName);
   }
+
+  @Test
+  public void suppress_expanded_fields() throws Exception {
+    dao = new MultiIndexDao(createDao());
+    dao.init(getAccessConfig());
+
+    Map<String, Object> fields = new HashMap<>();
+    fields.put("guid", "bro_1");
+    fields.put("source.type", SENSOR_NAME);
+    fields.put("ip_src_port", 8010);
+    fields.put("long_field", 10000);
+    fields.put("latitude", 48.5839);
+    fields.put("score", 10.0);
+    fields.put("is_alert", true);
+    fields.put("field.location_point", "48.5839,7.7455");
+
+    Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L);
+    dao.update(document, Optional.of(SENSOR_NAME));
+
+    Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
+
+    // assert no extra expanded fields are included
+    Assert.assertEquals(8, indexedDocument.getDocument().size());
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 85d14f4..1cf8c34 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -116,7 +116,12 @@ public class SolrComponent implements InMemoryComponent {
   @Override
   public void reset() {
     try {
-      miniSolrCloudCluster.deleteCollection("metron");
+      MetronSolrClient solr = getSolrClient();
+      for (String collection: collections.keySet()) {
+        solr.setDefaultCollection(collection);
+        solr.deleteByQuery("*:*");
+      }
+      solr.commit();
     } catch (SolrServerException | IOException e) {
       // Do nothing
     }


Mime
View raw message