metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [1/7] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
Date Wed, 23 May 2018 14:34:28 GMT
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr eb3366656 -> 49f851e0b


http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 1cf8c34..4bc9f8a 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -19,9 +19,13 @@ package org.apache.metron.solr.integration.components;
 
 import com.google.common.base.Function;
 import java.util.Collection;
+import java.util.Map.Entry;
 import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
 import org.apache.metron.integration.InMemoryComponent;
 import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.solr.dao.SolrUtilities;
 import org.apache.metron.solr.writer.MetronSolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
@@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.embedded.JettyConfig;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.cloud.MiniSolrCloudCluster;
 import org.apache.solr.common.SolrDocument;
 
@@ -40,13 +45,15 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.zookeeper.KeeperException;
 
 public class SolrComponent implements InMemoryComponent {
 
   public static class Builder {
+
     private int port = 8983;
     private String solrXmlPath = "../metron-solr/src/test/resources/solr/solr.xml";
-    private Map<String, String> collections = new HashMap<>();
+    private Map<String, String> initialCollections = new HashMap<>();
     private Function<SolrComponent, Void> postStartCallback;
 
     public Builder withPort(int port) {
@@ -59,8 +66,8 @@ public class SolrComponent implements InMemoryComponent {
       return this;
     }
 
-    public Builder addCollection(String name, String configPath) {
-      collections.put(name, configPath);
+    public Builder addInitialCollection(String name, String configPath) {
+      initialCollections.put(name, configPath);
       return this;
     }
 
@@ -69,9 +76,8 @@ public class SolrComponent implements InMemoryComponent {
       return this;
     }
 
-    public SolrComponent build() throws Exception {
-      if (collections.isEmpty()) throw new Exception("Must add at least 1 collection");
-      return new SolrComponent(port, solrXmlPath, collections, postStartCallback);
+    public SolrComponent build() {
+      return new SolrComponent(port, solrXmlPath, initialCollections, postStartCallback);
     }
   }
 
@@ -81,7 +87,8 @@ public class SolrComponent implements InMemoryComponent {
   private MiniSolrCloudCluster miniSolrCloudCluster;
   private Function<SolrComponent, Void> postStartCallback;
 
-  private SolrComponent(int port, String solrXmlPath, Map<String, String> collections,
Function<SolrComponent, Void> postStartCallback) throws Exception {
+  private SolrComponent(int port, String solrXmlPath, Map<String, String> collections,
+      Function<SolrComponent, Void> postStartCallback) {
     this.port = port;
     this.solrXmlPath = solrXmlPath;
     this.collections = collections;
@@ -93,14 +100,17 @@ public class SolrComponent implements InMemoryComponent {
     try {
       File baseDir = Files.createTempDirectory("solrcomponent").toFile();
       baseDir.deleteOnExit();
-      miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), JettyConfig.builder().setPort(port).build());
+      miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(),
+          JettyConfig.builder().setPort(port).build());
       for(String name: collections.keySet()) {
         String configPath = collections.get(name);
         miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
         CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient());
       }
-      if (postStartCallback != null) postStartCallback.apply(this);
-    } catch(Exception e) {
+      if (postStartCallback != null) {
+        postStartCallback.apply(this);
+      }
+    } catch (Exception e) {
       throw new UnableToStartException(e.getMessage(), e);
     }
   }
@@ -108,21 +118,18 @@ public class SolrComponent implements InMemoryComponent {
   @Override
   public void stop() {
     try {
+      miniSolrCloudCluster.deleteAllCollections();
       miniSolrCloudCluster.shutdown();
     } catch (Exception e) {
+      // Do nothing
     }
   }
 
   @Override
   public void reset() {
     try {
-      MetronSolrClient solr = getSolrClient();
-      for (String collection: collections.keySet()) {
-        solr.setDefaultCollection(collection);
-        solr.deleteByQuery("*:*");
-      }
-      solr.commit();
-    } catch (SolrServerException | IOException e) {
+      miniSolrCloudCluster.deleteAllCollections();
+    } catch (Exception e) {
       // Do nothing
     }
   }
@@ -139,12 +146,19 @@ public class SolrComponent implements InMemoryComponent {
     return miniSolrCloudCluster.getZkServer().getZkAddress();
   }
 
+  public void addCollection(String name, String configPath)
+      throws InterruptedException, IOException, KeeperException, SolrServerException {
+    miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
+    CollectionAdminRequest.createCollection(name, 1, 1)
+        .process(miniSolrCloudCluster.getSolrClient());
+  }
+
   public boolean hasCollection(String collection) {
     MetronSolrClient solr = getSolrClient();
     boolean collectionFound = false;
     try {
       collectionFound = solr.listCollections().contains(collection);
-    } catch(Exception e) {
+    } catch (Exception e) {
       e.printStackTrace();
     }
     return collectionFound;
@@ -155,12 +169,21 @@ public class SolrComponent implements InMemoryComponent {
     CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
     solr.setDefaultCollection(collection);
     SolrQuery parameters = new SolrQuery();
-    parameters.set("q", "*:*");
+
+    // If it's metaalert, we need to adjust the query. We want child docs with the parent,
+    // not separate.
+    if (collection.equals("metaalert")) {
+      parameters.setQuery("source.type:metaalert")
+          .setFields("*", "[child parentFilter=source.type:metaalert limit=999]");
+    } else {
+      parameters.set("q", "*:*");
+    }
     try {
       solr.commit();
       QueryResponse response = solr.query(parameters);
       for (SolrDocument solrDocument : response.getResults()) {
-        docs.add(solrDocument);
+        // Use the utils to make sure we get child docs.
+        docs.add(SolrUtilities.toDocument(solrDocument).getDocument());
       }
     } catch (SolrServerException | IOException e) {
       e.printStackTrace();
@@ -174,9 +197,36 @@ public class SolrComponent implements InMemoryComponent {
     solr.setDefaultCollection(collection);
     Collection<SolrInputDocument> solrInputDocuments = docs.stream().map(doc ->
{
       SolrInputDocument solrInputDocument = new SolrInputDocument();
-      doc.forEach(solrInputDocument::addField);
+      for (Entry<String, Object> entry : doc.entrySet()) {
+        // If the entry itself is a map, add it as a child document. Handle one level of
nesting.
+        if (entry.getValue() instanceof List && !entry.getKey().equals(
+            MetaAlertConstants.METAALERT_FIELD)) {
+          for (Object entryItem : (List)entry.getValue()) {
+            if (entryItem instanceof Map) {
+              @SuppressWarnings("unchecked")
+              Map<String, Object> childDoc = (Map<String, Object>) entryItem;
+              SolrInputDocument childInputDoc = new SolrInputDocument();
+              for (Entry<String, Object> childEntry : childDoc.entrySet()) {
+                childInputDoc.addField(childEntry.getKey(), childEntry.getValue());
+              }
+              solrInputDocument.addChildDocument(childInputDoc);
+            }
+          }
+        } else {
+          solrInputDocument.addField(entry.getKey(), entry.getValue());
+        }
+      }
       return solrInputDocument;
     }).collect(Collectors.toList());
-    solr.add(collection, solrInputDocuments);
+
+    checkUpdateResponse(solr.add(collection, solrInputDocuments));
+    // Make sure to commit so things show up
+    checkUpdateResponse(solr.commit(true, true));
+  }
+
+  protected void checkUpdateResponse(UpdateResponse result) throws IOException {
+    if (result.getStatus() != 0) {
+      throw new IOException("Response error received while adding documents: " + result);
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
index e655428..73113f6 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
@@ -25,9 +25,12 @@ import org.apache.metron.common.writer.BulkWriterResponse;
 import org.apache.metron.solr.integration.components.SolrComponent;
 import org.apache.metron.solr.writer.SolrWriter;
 import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.storm.tuple.Tuple;
+import org.apache.zookeeper.KeeperException;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mock;
 import java.io.File;
@@ -38,7 +41,6 @@ import java.util.*;
 import static org.mockito.Mockito.mock;
 
 public class SchemaValidationIntegrationTest {
-
   public static Iterable<String> getData(String sensor) throws IOException {
     return Iterables.filter(
             Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()),
@@ -53,9 +55,7 @@ public class SchemaValidationIntegrationTest {
   }
 
   public static SolrComponent createSolrComponent(String sensor) throws Exception {
-    return new SolrComponent.Builder()
-            .addCollection(String.format("%s", sensor), String.format("src/main/config/schema/%s",
sensor))
-            .build();
+    return new SolrComponent.Builder().build();
   }
 
   @Test
@@ -92,6 +92,7 @@ public class SchemaValidationIntegrationTest {
     try {
       component = createSolrComponent(sensorType);
       component.start();
+      component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s",
sensorType));
       Map<String, Object> globalConfig = getGlobalConfig(sensorType, component);
 
       List<JSONObject> inputs = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
index 685c5fd..7b7d208 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
@@ -17,11 +17,19 @@
  */
 package org.apache.metron.solr.writer;
 
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
 import org.apache.metron.common.configuration.IndexingConfigurations;
 import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration;
 import org.apache.metron.enrichment.integration.utils.SampleUtil;
@@ -34,17 +42,6 @@ import org.junit.Test;
 import org.mockito.ArgumentMatcher;
 import org.mockito.Mockito;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.argThat;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
 
 public class SolrWriterTest {
 

http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
index 85d6915..bb2de59 100644
--- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
+++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
@@ -16,53 +16,59 @@
  limitations under the License.
 -->
 <schema name="example" version="1.6">
-   <field name="_version_" type="plong" indexed="false" stored="false"/>
+  <field name="_version_" type="plong" indexed="false" stored="false"/>
 
-   <!-- points to the root document of a block of nested documents. Required for nested
-      document support, may be removed otherwise
-   -->
-   <field name="_root_" type="string" indexed="true" stored="false" docValues="false"
/>
+  <!-- points to the root document of a block of nested documents. Required for nested
+     document support, may be removed otherwise
+  -->
+  <field name="_root_" type="string" indexed="true" stored="false" docValues="false"/>
 
-   <!-- Only remove the "id" field if you have a very good reason to. While not strictly
-     required, it is highly recommended. A <uniqueKey> is present in almost all Solr
-     installations. See the <uniqueKey> declaration below where <uniqueKey> is
set to "id".
-     Do NOT change the type and apply index-time analysis to the <uniqueKey> as it
will likely
-     make routing in SolrCloud and document replacement in general fail. Limited _query_
time
-     analysis is possible as long as the indexing process is guaranteed to index the term
-     in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce
multiple
-     tokens
-   -->
-   <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false"
/>
+  <!-- Only remove the "id" field if you have a very good reason to. While not strictly
+    required, it is highly recommended. A <uniqueKey> is present in almost all Solr
+    installations. See the <uniqueKey> declaration below where <uniqueKey> is
set to "id".
+    Do NOT change the type and apply index-time analysis to the <uniqueKey> as it will
likely
+    make routing in SolrCloud and document replacement in general fail. Limited _query_ time
+    analysis is possible as long as the indexing process is guaranteed to index the term
+    in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce
multiple
+    tokens
+  -->
+  <field name="guid" type="string" indexed="true" stored="true" required="true"
+    multiValued="false"/>
 
-   <field name="source:type" type="string" indexed="true" stored="true"/>
-   <field name="name" type="string" indexed="true" stored="true" />
-   <field name="timestamp" type="plong" indexed="true" stored="true" />
-   <field name="new-field" type="string" indexed="true" stored="true" />
+  <field name="source.type" type="string" indexed="true" stored="true"/>
+  <field name="name" type="string" indexed="true" stored="true"/>
+  <field name="timestamp" type="plong" indexed="true" stored="true"/>
+  <field name="new-field" type="string" indexed="true" stored="true"/>
+  <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
+  <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/>
+  <field name="score" type="pdouble" indexed="true" stored="true"/>
 
-   <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
 
+  <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
 
- <!-- Field to use to determine and enforce document uniqueness. 
-      Unless this field is marked with required="false", it will be a required field
-   -->
-    <uniqueKey>guid</uniqueKey>
 
-
-    <!-- field type definitions. The "name" attribute is
-       just a label to be used by field definitions.  The "class"
-       attribute and any other attributes determine the real
-       behavior of the fieldType.
-         Class names starting with "solr" refer to java classes in a
-       standard package such as org.apache.solr.analysis
+  <!-- Field to use to determine and enforce document uniqueness.
+       Unless this field is marked with required="false", it will be a required field
     -->
+  <uniqueKey>guid</uniqueKey>
+
+
+  <!-- field type definitions. The "name" attribute is
+     just a label to be used by field definitions.  The "class"
+     attribute and any other attributes determine the real
+     behavior of the fieldType.
+       Class names starting with "solr" refer to java classes in a
+     standard package such as org.apache.solr.analysis
+  -->
 
-    <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
-    <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
-    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
-    <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
-    <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
-    <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
-    <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
-    <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField"
/>
+  <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
+  <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+  <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+  <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
+  <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
+  <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
+  <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
+  <fieldType name="ignored" stored="false" indexed="false" multiValued="true"
+    class="solr.StrField"/>
 
 </schema>


Mime
View raw message