metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [1/2] metron git commit: METRON-1022: Elasticsearch REST endpoint this closes apache/incubator-metron#636
Date Thu, 20 Jul 2017 09:32:54 GMT
Repository: metron
Updated Branches:
  refs/heads/master 0d1923f83 -> cf7043c59


http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
new file mode 100644
index 0000000..ae4f9bd
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+import java.util.Map;
+
+public class SearchResult {
+
+  private String id;
+  private Map<String, Object> source;
+  private float score;
+
+  public String getId() {
+    return id;
+  }
+
+  public void setId(String id) {
+    this.id = id;
+  }
+
+  public Map<String, Object> getSource() {
+    return source;
+  }
+
+  public void setSource(Map<String, Object> source) {
+    this.source = source;
+  }
+
+  public float getScore() {
+    return score;
+  }
+
+  public void setScore(float score) {
+    this.score = score;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
new file mode 100644
index 0000000..a3473fc
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortField.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+public class SortField {
+  private String field;
+  private SortOrder sortOrder;
+
+  public String getField() {
+    return field;
+  }
+
+  public void setField(String field) {
+    this.field = field;
+  }
+
+  public SortOrder getSortOrder() {
+    return sortOrder;
+  }
+
+  public void setSortOrder(String sortOrder) {
+    this.sortOrder = SortOrder.fromString(sortOrder);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
new file mode 100644
index 0000000..fde3279
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SortOrder.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public enum SortOrder {
+  @JsonProperty("desc")
+  DESC("desc"),
+  @JsonProperty("asc")
+  ASC("asc");
+
+  private String sortOrder;
+
+  SortOrder(String sortOrder) {
+    this.sortOrder = sortOrder;
+  }
+
+  public String getSortOrder() {
+    return sortOrder;
+  }
+
+  public static SortOrder fromString(String order) {
+    return SortOrder.valueOf(order.toUpperCase());
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
new file mode 100644
index 0000000..444a9da
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.search.*;
+
+import java.io.IOException;
+import java.util.*;
+
+public class InMemoryDao implements IndexDao {
+  public static Map<String, List<String>> BACKING_STORE = new HashMap<>();
+  private AccessConfig config;
+
+  @Override
+  public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException
{
+    if(config.getMaxSearchResults() != null && searchRequest.getSize() > config.getMaxSearchResults())
{
+      throw new InvalidSearchException("Search result size must be less than " + config.getMaxSearchResults());
+    }
+    List<SearchResult> response = new ArrayList<>();
+    for(String index : searchRequest.getIndices()) {
+      String i = null;
+      for(String storedIdx : BACKING_STORE.keySet()) {
+        if(storedIdx.equals(index) || storedIdx.startsWith(index + "_")) {
+          i = storedIdx;
+        }
+      }
+      if(i == null) {
+        continue;
+      }
+      for (String doc : BACKING_STORE.get(i)) {
+        Map<String, Object> docParsed = parse(doc);
+        if (isMatch(searchRequest.getQuery(), docParsed)) {
+          SearchResult result = new SearchResult();
+          result.setSource(docParsed);
+          result.setScore((float) Math.random());
+          result.setId(docParsed.getOrDefault(Constants.GUID, UUID.randomUUID()).toString());
+          response.add(result);
+        }
+      }
+    }
+
+    if(searchRequest.getSort().size() != 0) {
+      Collections.sort(response, sorted(searchRequest.getSort()));
+    }
+    SearchResponse ret = new SearchResponse();
+    List<SearchResult> finalResp = new ArrayList<>();
+    int maxSize = config.getMaxSearchResults() == null?searchRequest.getSize():config.getMaxSearchResults();
+    for(int i = searchRequest.getFrom();i < response.size()&& finalResp.size()
<= maxSize;++i) {
+      finalResp.add(response.get(i));
+    }
+    ret.setTotal(response.size());
+    ret.setResults(finalResp);
+    return ret;
+  }
+
+
+  private static class ComparableComparator implements Comparator<Comparable>  {
+    SortOrder order = null;
+    public ComparableComparator(SortOrder order) {
+      this.order = order;
+    }
+    @Override
+    public int compare(Comparable o1, Comparable o2) {
+      int result = ComparisonChain.start().compare(o1, o2).result();
+      return order == SortOrder.ASC?result:-1*result;
+    }
+  }
+
+  private static Comparator<SearchResult> sorted(final List<SortField> fields)
{
+    return (o1, o2) -> {
+      ComparisonChain chain = ComparisonChain.start();
+      for(SortField field : fields) {
+        Comparable f1 = (Comparable) o1.getSource().get(field.getField());
+        Comparable f2 = (Comparable) o2.getSource().get(field.getField());
+        chain = chain.compare(f1, f2, new ComparableComparator(field.getSortOrder()));
+      }
+      return chain.result();
+    };
+  }
+
+  private static boolean isMatch(String query, Map<String, Object> doc) {
+    if(query.equals("*")) {
+      return true;
+    }
+    if(query.contains(":")) {
+      Iterable<String> splits = Splitter.on(":").split(query.trim());
+      String field = Iterables.getFirst(splits, "");
+      String val = Iterables.getLast(splits, "");
+      Object o = doc.get(field);
+      if(o == null) {
+        return false;
+      }
+      else {
+        return o.equals(val);
+      }
+    }
+    return false;
+  }
+
+  private static Map<String, Object> parse(String doc) {
+    try {
+      return JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>()
{});
+    } catch (IOException e) {
+      throw new IllegalStateException(e.getMessage(), e);
+    }
+
+  }
+
+  @Override
+  public void init(Map<String, Object> globalConfig, AccessConfig config) {
+    this.config = config;
+  }
+
+  public static void load(Map<String, List<String>> backingStore) {
+    BACKING_STORE = backingStore;
+  }
+
+  public static void clear() {
+    BACKING_STORE.clear();
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java
b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java
new file mode 100644
index 0000000..8b5baef
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.integration.InMemoryComponent;
+import org.json.simple.parser.ParseException;
+import org.junit.*;
+
+import java.util.List;
+
+public abstract class IndexingDaoIntegrationTest {
+  /**
+   * [
+   * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "timestamp":1,
"rejected":true},
+   * {"source:type": "bro" "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "timestamp":2,
"rejected":false},
+   * {"source:type": "bro" "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "timestamp":3,
"rejected":true},
+   * {"source:type": "bro" "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "timestamp":4,
"rejected":false},
+   * {"source:type": "bro" "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "timestamp":5,
"rejected":true}
+   * ]
+   */
+  @Multiline
+  public static String broData;
+
+  /**
+   * [
+   * {"source:type": "snort" "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "timestamp":6,
"is_alert":false},
+   * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "timestamp":7,
"is_alert":true},
+   * {"source:type": "snort" "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "timestamp":8,
"is_alert":false},
+   * {"source:type": "snort" "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "timestamp":9,
"is_alert":true},
+   * {"source:type": "snort" "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "timestamp":10,
"is_alert":false}
+   * ]
+   */
+  @Multiline
+  public static String snortData;
+
+  /**
+   * {
+   * "indices": ["bro", "snort"],
+   * "query": "*",
+   * "from": 0,
+   * "size": 10,
+   * "sort": [
+   *   {
+   *     "field": "timestamp",
+   *     "sortOrder": "desc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String allQuery;
+
+  /**
+   * {
+   * "indices": ["bro", "snort"],
+   * "query": "ip_src_addr:192.168.1.1",
+   * "from": 0,
+   * "size": 10,
+   * "sort": [
+   *   {
+   *     "field": "timestamp",
+   *     "sortOrder": "desc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String filterQuery;
+
+  /**
+   * {
+   * "indices": ["bro", "snort"],
+   * "query": "*",
+   * "from": 0,
+   * "size": 10,
+   * "sort": [
+   *   {
+   *     "field": "ip_src_port",
+   *     "sortOrder": "asc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String sortQuery;
+
+  /**
+   * {
+   * "indices": ["bro", "snort"],
+   * "query": "*",
+   * "from": 4,
+   * "size": 3,
+   * "sort": [
+   *   {
+   *     "field": "timestamp",
+   *     "sortOrder": "desc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String paginationQuery;
+
+  /**
+   * {
+   * "indices": ["bro"],
+   * "query": "*",
+   * "from": 0,
+   * "size": 10,
+   * "sort": [
+   *   {
+   *     "field": "timestamp",
+   *     "sortOrder": "desc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String indexQuery;
+
+  /**
+   * {
+   * "indices": ["bro", "snort"],
+   * "query": "*",
+   * "from": 0,
+   * "size": 101,
+   * "sort": [
+   *   {
+   *     "field": "timestamp",
+   *     "sortOrder": "desc"
+   *   }
+   * ]
+   * }
+   */
+  @Multiline
+  public static String exceededMaxResultsQuery;
+
+  protected IndexDao dao;
+  protected InMemoryComponent indexComponent;
+
+  @Before
+  public void setup() throws Exception {
+    indexComponent = startIndex();
+    loadTestData();
+    dao = createDao();
+  }
+
+  @Test
+  public void test() throws Exception {
+    //All Query Testcase
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
+      SearchResponse response = dao.search(request);
+      Assert.assertEquals(10, response.getTotal());
+      List<SearchResult> results = response.getResults();
+      for(int i = 0;i < 5;++i) {
+        Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+        Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp"));
+      }
+      for(int i = 5;i < 10;++i) {
+        Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+        Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp"));
+      }
+    }
+    //Filter test case
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
+      SearchResponse response = dao.search(request);
+      Assert.assertEquals(3, response.getTotal());
+      List<SearchResult> results = response.getResults();
+      Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+      Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
+      Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
+      Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
+      Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+      Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
+    }
+    //Sort test case
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
+      SearchResponse response = dao.search(request);
+      Assert.assertEquals(10, response.getTotal());
+      List<SearchResult> results = response.getResults();
+      for(int i = 8001;i < 8011;++i) {
+        Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port"));
+      }
+    }
+    //pagination test case
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
+      SearchResponse response = dao.search(request);
+      Assert.assertEquals(10, response.getTotal());
+      List<SearchResult> results = response.getResults();
+      Assert.assertEquals(3, results.size());
+      Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+      Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
+      Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
+      Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
+      Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+      Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
+    }
+    //Index query
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
+      SearchResponse response = dao.search(request);
+      Assert.assertEquals(5, response.getTotal());
+      List<SearchResult> results = response.getResults();
+      for(int i = 5,j=0;i > 0;i--,j++) {
+        Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
+        Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
+      }
+    }
+    //Exceeded maximum results query
+    {
+      SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
+      try {
+        dao.search(request);
+        Assert.fail("Exception expected, but did not come.");
+      }
+      catch(InvalidSearchException ise) {
+        Assert.assertEquals("Search result size must be less than 100", ise.getMessage());
+      }
+    }
+  }
+
+  @After
+  public void stop() throws Exception {
+    indexComponent.stop();
+  }
+
+  protected abstract IndexDao createDao() throws Exception;
+  protected abstract InMemoryComponent startIndex() throws Exception;
+  protected abstract void loadTestData() throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cf7043c5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5079655..5d64019 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,6 +111,7 @@
         <global_maven_version>[3.3.1,)</global_maven_version>
         <global_kryo_version>3.0.3</global_kryo_version>
         <global_kryo_serializers_version>0.38</global_kryo_serializers_version>
+        <global_reflections_version>0.9.10</global_reflections_version>
         <argLine></argLine>
     </properties>
 


Mime
View raw message