From commits-return-3395-archive-asf-public=cust-asf.ponee.io@metron.apache.org Wed Jul 11 03:32:20 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 55D7518078F for ; Wed, 11 Jul 2018 03:32:19 +0200 (CEST) Received: (qmail 93950 invoked by uid 500); 11 Jul 2018 01:32:18 -0000 Mailing-List: contact commits-help@metron.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.apache.org Delivered-To: mailing list commits@metron.apache.org Received: (qmail 93839 invoked by uid 99); 11 Jul 2018 01:32:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Jul 2018 01:32:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id ED500E0255; Wed, 11 Jul 2018 01:32:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mmiklavcic@apache.org To: commits@metron.apache.org Date: Wed, 11 Jul 2018 01:32:22 -0000 Message-Id: <05e703e76e624231a56efddf626fc24e@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [06/50] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970 http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java index 1cf8c34..4bc9f8a 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java @@ -19,9 +19,13 @@ package org.apache.metron.solr.integration.components; import com.google.common.base.Function; import java.util.Collection; +import java.util.Map.Entry; import java.util.stream.Collectors; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; +import org.apache.metron.solr.dao.SolrUtilities; import org.apache.metron.solr.writer.MetronSolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; @@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.SolrDocument; @@ -40,13 +45,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; +import org.apache.zookeeper.KeeperException; public class SolrComponent implements InMemoryComponent { public static class Builder { + private int port = 8983; private String solrXmlPath = "../metron-solr/src/test/resources/solr/solr.xml"; - private Map collections = new HashMap<>(); + private Map initialCollections = new HashMap<>(); private Function postStartCallback; public Builder withPort(int port) { @@ -59,8 +66,8 @@ public class SolrComponent implements InMemoryComponent { return this; } - public Builder addCollection(String name, String configPath) { - collections.put(name, configPath); + public Builder addInitialCollection(String name, String configPath) { + initialCollections.put(name, configPath); return this; } @@ -69,9 +76,8 @@ public class SolrComponent implements InMemoryComponent { return this; } - public SolrComponent build() throws Exception { - if (collections.isEmpty()) throw new Exception("Must add at least 1 collection"); - return new SolrComponent(port, solrXmlPath, collections, postStartCallback); + public SolrComponent build() { + return new SolrComponent(port, solrXmlPath, initialCollections, postStartCallback); } } @@ -81,7 +87,8 @@ public class SolrComponent implements InMemoryComponent { private MiniSolrCloudCluster miniSolrCloudCluster; private Function postStartCallback; - private SolrComponent(int port, String solrXmlPath, Map collections, Function postStartCallback) throws Exception { + private SolrComponent(int port, String solrXmlPath, Map collections, + Function postStartCallback) { this.port = port; this.solrXmlPath = solrXmlPath; this.collections = collections; @@ -93,14 +100,17 @@ public class SolrComponent implements InMemoryComponent { try { File baseDir = Files.createTempDirectory("solrcomponent").toFile(); baseDir.deleteOnExit(); - miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), JettyConfig.builder().setPort(port).build()); + miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), + JettyConfig.builder().setPort(port).build()); for(String name: collections.keySet()) { String configPath = collections.get(name); miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient()); } - if (postStartCallback != null) postStartCallback.apply(this); - } catch(Exception e) { + if (postStartCallback != null) { + postStartCallback.apply(this); + } + } catch (Exception e) { throw new UnableToStartException(e.getMessage(), e); } } @@ -108,21 +118,18 @@ public class SolrComponent implements InMemoryComponent { @Override public void stop() { try { + miniSolrCloudCluster.deleteAllCollections(); miniSolrCloudCluster.shutdown(); } catch (Exception e) { + // Do nothing } } @Override public void reset() { try { - MetronSolrClient solr = getSolrClient(); - for (String collection: collections.keySet()) { - solr.setDefaultCollection(collection); - solr.deleteByQuery("*:*"); - } - solr.commit(); - } catch (SolrServerException | IOException e) { + miniSolrCloudCluster.deleteAllCollections(); + } catch (Exception e) { // Do nothing } } @@ -139,12 +146,19 @@ public class SolrComponent implements InMemoryComponent { return miniSolrCloudCluster.getZkServer().getZkAddress(); } + public void addCollection(String name, String configPath) + throws InterruptedException, IOException, KeeperException, SolrServerException { + miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); + CollectionAdminRequest.createCollection(name, 1, 1) + .process(miniSolrCloudCluster.getSolrClient()); + } + public boolean hasCollection(String collection) { MetronSolrClient solr = getSolrClient(); boolean collectionFound = false; try { collectionFound = solr.listCollections().contains(collection); - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); } return collectionFound; @@ -155,12 +169,21 @@ public class SolrComponent implements InMemoryComponent { CloudSolrClient solr = miniSolrCloudCluster.getSolrClient(); solr.setDefaultCollection(collection); SolrQuery parameters = new SolrQuery(); - parameters.set("q", "*:*"); + + // If it's metaalert, we need to adjust the query. We want child docs with the parent, + // not separate. + if (collection.equals("metaalert")) { + parameters.setQuery("source.type:metaalert") + .setFields("*", "[child parentFilter=source.type:metaalert limit=999]"); + } else { + parameters.set("q", "*:*"); + } try { solr.commit(); QueryResponse response = solr.query(parameters); for (SolrDocument solrDocument : response.getResults()) { - docs.add(solrDocument); + // Use the utils to make sure we get child docs. + docs.add(SolrUtilities.toDocument(solrDocument).getDocument()); } } catch (SolrServerException | IOException e) { e.printStackTrace(); @@ -174,9 +197,36 @@ public class SolrComponent implements InMemoryComponent { solr.setDefaultCollection(collection); Collection solrInputDocuments = docs.stream().map(doc -> { SolrInputDocument solrInputDocument = new SolrInputDocument(); - doc.forEach(solrInputDocument::addField); + for (Entry entry : doc.entrySet()) { + // If the entry itself is a map, add it as a child document. Handle one level of nesting. + if (entry.getValue() instanceof List && !entry.getKey().equals( + MetaAlertConstants.METAALERT_FIELD)) { + for (Object entryItem : (List)entry.getValue()) { + if (entryItem instanceof Map) { + @SuppressWarnings("unchecked") + Map childDoc = (Map) entryItem; + SolrInputDocument childInputDoc = new SolrInputDocument(); + for (Entry childEntry : childDoc.entrySet()) { + childInputDoc.addField(childEntry.getKey(), childEntry.getValue()); + } + solrInputDocument.addChildDocument(childInputDoc); + } + } + } else { + solrInputDocument.addField(entry.getKey(), entry.getValue()); + } + } return solrInputDocument; }).collect(Collectors.toList()); - solr.add(collection, solrInputDocuments); + + checkUpdateResponse(solr.add(collection, solrInputDocuments)); + // Make sure to commit so things show up + checkUpdateResponse(solr.commit(true, true)); + } + + protected void checkUpdateResponse(UpdateResponse result) throws IOException { + if (result.getStatus() != 0) { + throw new IOException("Response error received while adding documents: " + result); + } } } http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java index e655428..73113f6 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java @@ -25,9 +25,12 @@ import org.apache.metron.common.writer.BulkWriterResponse; import org.apache.metron.solr.integration.components.SolrComponent; import org.apache.metron.solr.writer.SolrWriter; import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.storm.tuple.Tuple; +import org.apache.zookeeper.KeeperException; import org.json.simple.JSONObject; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import java.io.File; @@ -38,7 +41,6 @@ import java.util.*; import static org.mockito.Mockito.mock; public class SchemaValidationIntegrationTest { - public static Iterable getData(String sensor) throws IOException { return Iterables.filter( Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()), @@ -53,9 +55,7 @@ public class SchemaValidationIntegrationTest { } public static SolrComponent createSolrComponent(String sensor) throws Exception { - return new SolrComponent.Builder() - .addCollection(String.format("%s", sensor), String.format("src/main/config/schema/%s", sensor)) - .build(); + return new SolrComponent.Builder().build(); } @Test @@ -92,6 +92,7 @@ public class SchemaValidationIntegrationTest { try { component = createSolrComponent(sensorType); component.start(); + component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType)); Map globalConfig = getGlobalConfig(sensorType, component); List inputs = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java index 685c5fd..7b7d208 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java @@ -17,11 +17,19 @@ */ package org.apache.metron.solr.writer; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration; import org.apache.metron.enrichment.integration.utils.SampleUtil; @@ -34,17 +42,6 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.argThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - public class SolrWriterTest { http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema index 85d6915..bb2de59 100644 --- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema +++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema @@ -16,53 +16,59 @@ limitations under the License. --> - + - - + + - - + + - - - - + + + + + + + - + - - guid - - + guid + + + - - - - - - - - + + + + + + + +