Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C7258200CCB for ; Thu, 20 Jul 2017 11:32:57 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id C58EB16AF0C; Thu, 20 Jul 2017 09:32:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 95CD416AF07 for ; Thu, 20 Jul 2017 11:32:56 +0200 (CEST) Received: (qmail 15173 invoked by uid 500); 20 Jul 2017 09:32:55 -0000 Mailing-List: contact commits-help@metron.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.apache.org Delivered-To: mailing list commits@metron.apache.org Received: (qmail 15163 invoked by uid 99); 20 Jul 2017 09:32:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 20 Jul 2017 09:32:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8ECC8E2F41; Thu, 20 Jul 2017 09:32:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cestella@apache.org To: commits@metron.apache.org Date: Thu, 20 Jul 2017 09:32:54 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] metron git commit: METRON-1022: Elasticsearch REST endpoint this closes apache/incubator-metron#636 archived-at: Thu, 20 Jul 2017 09:32:58 -0000 Repository: metron Updated Branches: refs/heads/master 0d1923f83 -> cf7043c59 http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java new file mode 100644 index 0000000..ae4f9bd --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +import java.util.Map; + +public class SearchResult { + + private String id; + private Map source; + private float score; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source = source; + } + + public float getScore() { + return score; + } + + public void setScore(float score) { + this.score = score; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java new file mode 100644 index 0000000..a3473fc --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +public class SortField { + private String field; + private SortOrder sortOrder; + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public SortOrder getSortOrder() { + return sortOrder; + } + + public void setSortOrder(String sortOrder) { + this.sortOrder = SortOrder.fromString(sortOrder); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java new file mode 100644 index 0000000..fde3279 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao.search; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum SortOrder { + @JsonProperty("desc") + DESC("desc"), + @JsonProperty("asc") + ASC("asc"); + + private String sortOrder; + + SortOrder(String sortOrder) { + this.sortOrder = sortOrder; + } + + public String getSortOrder() { + return sortOrder; + } + + public static SortOrder fromString(String order) { + return SortOrder.valueOf(order.toUpperCase()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java new file mode 100644 index 0000000..444a9da --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Splitter; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Iterables; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.*; + +import java.io.IOException; +import java.util.*; + +public class InMemoryDao implements IndexDao { + public static Map> BACKING_STORE = new HashMap<>(); + private AccessConfig config; + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + if(config.getMaxSearchResults() != null && searchRequest.getSize() > config.getMaxSearchResults()) { + throw new InvalidSearchException("Search result size must be less than " + config.getMaxSearchResults()); + } + List response = new ArrayList<>(); + for(String index : searchRequest.getIndices()) { + String i = null; + for(String storedIdx : BACKING_STORE.keySet()) { + if(storedIdx.equals(index) || storedIdx.startsWith(index + "_")) { + i = storedIdx; + } + } + if(i == null) { + continue; + } + for (String doc : BACKING_STORE.get(i)) { + Map docParsed = parse(doc); + if (isMatch(searchRequest.getQuery(), docParsed)) { + SearchResult result = new SearchResult(); + result.setSource(docParsed); + result.setScore((float) Math.random()); + result.setId(docParsed.getOrDefault(Constants.GUID, UUID.randomUUID()).toString()); + response.add(result); + } + } + } + + if(searchRequest.getSort().size() != 0) { + Collections.sort(response, sorted(searchRequest.getSort())); + } + SearchResponse ret = new SearchResponse(); + List finalResp = new ArrayList<>(); + int maxSize = config.getMaxSearchResults() == null?searchRequest.getSize():config.getMaxSearchResults(); + for(int i = searchRequest.getFrom();i < response.size()&& finalResp.size() <= maxSize;++i) { + finalResp.add(response.get(i)); + } + ret.setTotal(response.size()); + ret.setResults(finalResp); + return ret; + } + + + private static class ComparableComparator implements Comparator { + SortOrder order = null; + public ComparableComparator(SortOrder order) { + this.order = order; + } + @Override + public int compare(Comparable o1, Comparable o2) { + int result = ComparisonChain.start().compare(o1, o2).result(); + return order == SortOrder.ASC?result:-1*result; + } + } + + private static Comparator sorted(final List fields) { + return (o1, o2) -> { + ComparisonChain chain = ComparisonChain.start(); + for(SortField field : fields) { + Comparable f1 = (Comparable) o1.getSource().get(field.getField()); + Comparable f2 = (Comparable) o2.getSource().get(field.getField()); + chain = chain.compare(f1, f2, new ComparableComparator(field.getSortOrder())); + } + return chain.result(); + }; + } + + private static boolean isMatch(String query, Map doc) { + if(query.equals("*")) { + return true; + } + if(query.contains(":")) { + Iterable splits = Splitter.on(":").split(query.trim()); + String field = Iterables.getFirst(splits, ""); + String val = Iterables.getLast(splits, ""); + Object o = doc.get(field); + if(o == null) { + return false; + } + else { + return o.equals(val); + } + } + return false; + } + + private static Map parse(String doc) { + try { + return JSONUtils.INSTANCE.load(doc, new TypeReference>() {}); + } catch (IOException e) { + throw new IllegalStateException(e.getMessage(), e); + } + + } + + @Override + public void init(Map globalConfig, AccessConfig config) { + this.config = config; + } + + public static void load(Map> backingStore) { + BACKING_STORE = backingStore; + } + + public static void clear() { + BACKING_STORE.clear(); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java new file mode 100644 index 0000000..8b5baef --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.indexing.dao; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.integration.InMemoryComponent; +import org.json.simple.parser.ParseException; +import org.junit.*; + +import java.util.List; + +public abstract class IndexingDaoIntegrationTest { + /** + * [ + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "timestamp":1, "rejected":true}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "timestamp":2, "rejected":false}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "timestamp":3, "rejected":true}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "timestamp":4, "rejected":false}, + * {"source:type": "bro" "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "timestamp":5, "rejected":true} + * ] + */ + @Multiline + public static String broData; + + /** + * [ + * {"source:type": "snort" "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "timestamp":6, "is_alert":false}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "timestamp":7, "is_alert":true}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "timestamp":8, "is_alert":false}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "timestamp":9, "is_alert":true}, + * {"source:type": "snort" "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "timestamp":10, "is_alert":false} + * ] + */ + @Multiline + public static String snortData; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String allQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "ip_src_addr:192.168.1.1", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String filterQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "ip_src_port", + * "sortOrder": "asc" + * } + * ] + * } + */ + @Multiline + public static String sortQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 4, + * "size": 3, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String paginationQuery; + + /** + * { + * "indices": ["bro"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String indexQuery; + + /** + * { + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 101, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String exceededMaxResultsQuery; + + protected IndexDao dao; + protected InMemoryComponent indexComponent; + + @Before + public void setup() throws Exception { + indexComponent = startIndex(); + loadTestData(); + dao = createDao(); + } + + @Test + public void test() throws Exception { + //All Query Testcase + { + SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for(int i = 0;i < 5;++i) { + Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + } + for(int i = 5;i < 10;++i) { + Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); + } + } + //Filter test case + { + SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(3, response.getTotal()); + List results = response.getResults(); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + } + //Sort test case + { + SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for(int i = 8001;i < 8011;++i) { + Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port")); + } + } + //pagination test case + { + SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + Assert.assertEquals(3, results.size()); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + //Index query + { + SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List results = response.getResults(); + for(int i = 5,j=0;i > 0;i--,j++) { + Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); + } + } + //Exceeded maximum results query + { + SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); + try { + dao.search(request); + Assert.fail("Exception expected, but did not come."); + } + catch(InvalidSearchException ise) { + Assert.assertEquals("Search result size must be less than 100", ise.getMessage()); + } + } + } + + @After + public void stop() throws Exception { + indexComponent.stop(); + } + + protected abstract IndexDao createDao() throws Exception; + protected abstract InMemoryComponent startIndex() throws Exception; + protected abstract void loadTestData() throws Exception; +} http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5079655..5d64019 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,7 @@ [3.3.1,) 3.0.3 0.38 + 0.9.10