metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-1243: Add a REST endpoint which allows us to get a list of all indice closes apache/incubator-metron#797
Date Wed, 25 Oct 2017 18:48:24 GMT
Repository: metron
Updated Branches:
  refs/heads/master a25de0fc1 -> 1c12e22df


METRON-1243: Add a REST endpoint which allows us to get a list of all indice closes apache/incubator-metron#797


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c12e22d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c12e22d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c12e22d

Branch: refs/heads/master
Commit: 1c12e22df03b53981ca8cef426ab26f263f289c0
Parents: a25de0f
Author: cstella <cestella@gmail.com>
Authored: Wed Oct 25 14:48:17 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Wed Oct 25 14:48:17 2017 -0400

----------------------------------------------------------------------
 .../SensorIndexingConfigController.java         |  8 ++
 .../service/SensorIndexingConfigService.java    |  2 +
 .../impl/SensorIndexingConfigServiceImpl.java   | 34 +++++++-
 ...IndexingConfigControllerIntegrationTest.java | 15 ++++
 .../SensorIndexingConfigServiceImplTest.java    | 83 +++++++++++++++++++-
 5 files changed, 140 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
index ffd80fb..ff1d6bd 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
@@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.List;
 import java.util.Map;
 
 @RestController
@@ -74,6 +75,13 @@ public class SensorIndexingConfigController {
     return new ResponseEntity<>(sensorIndexingConfigService.getAll(), HttpStatus.OK);
   }
 
+  @ApiOperation(value = "Retrieves all indices")
+  @ApiResponse(message = "Returns all the indices in use", code = 200)
+  @RequestMapping(value = "/list/indices/{writerName}", method = RequestMethod.GET)
+  ResponseEntity<Iterable<String>> getAllIndices(@ApiParam(name="writerName",
value="Writer name.  One of solr, elasticsearch or hdfs", required=true)@PathVariable String
writerName) throws Exception {
+    return new ResponseEntity<>(sensorIndexingConfigService.getAllIndices(writerName),
HttpStatus.OK);
+  }
+
   @ApiOperation(value = "Deletes a SensorIndexingConfig from Zookeeper")
   @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig was deleted", code
= 200),
           @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) })

http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
index 0699431..0f32e53 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
@@ -32,6 +32,8 @@ public interface SensorIndexingConfigService {
 
   List<String> getAllTypes() throws RestException;
 
+  Iterable<String> getAllIndices(String writerName) throws RestException;
+
   boolean delete(String name) throws RestException;
 
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index 5c73b26..a90d123 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -18,14 +18,16 @@
 package org.apache.metron.rest.service.impl;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.metron.common.configuration.ConfigurationType;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
 import org.apache.metron.common.zookeeper.ConfigurationsCache;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.service.SensorIndexingConfigService;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
 import org.apache.zookeeper.KeeperException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -33,6 +35,9 @@ import org.springframework.stereotype.Service;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Collections;
+import java.util.Set;
+import java.util.HashSet;
 
 @Service
 public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
@@ -85,6 +90,33 @@ public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigServ
     return configs.getTypes();
   }
 
+  /**
+   * Get a list of index names for a given writer (e.g. elasticsearch, solr, hdfs).
+   * This functions in the following way:
+   *   * If an index config exists, then the index name will be returned. If unspecified,
then the sensor name is used
+   *   * If a parser exists and an index does NOT exist, then it will be included.
+   *   * If the writer is disabled in the index config, then it will NOT be included.
+   * @param writerName The writer name to use
+   * @return An iterable of index names
+   * @throws RestException
+   */
+  @Override
+  public Iterable<String> getAllIndices(String writerName) throws RestException {
+    if(StringUtils.isEmpty(writerName)) {
+      return Collections.emptyList();
+    }
+    IndexingConfigurations indexingConfigs = cache.get( IndexingConfigurations.class);
+    ParserConfigurations parserConfigs = cache.get( ParserConfigurations.class);
+    Set<String> ret = new HashSet<>();
+    for(String sensorName : Iterables.concat(parserConfigs.getTypes(), indexingConfigs.getTypes()))
{
+      if(indexingConfigs.isEnabled(sensorName, writerName)) {
+        String indexName = indexingConfigs.getIndex(sensorName, writerName);
+        ret.add(indexName == null ? sensorName : indexName);
+      }
+    }
+    return ret;
+  }
+
   @Override
   public boolean delete(String name) throws RestException {
     try {

http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index 674c55a..28977fd 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -98,12 +98,27 @@ public class SensorIndexingConfigControllerIntegrationTest {
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
             .andExpect(content().bytes("{}".getBytes()));
 
+    this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
+            .andExpect(status().isOk())
+            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+            .andExpect(content().bytes("[]".getBytes()));
+
+    this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/blah").with(httpBasic(user,password)))
+            .andExpect(status().isOk())
+            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+            .andExpect(content().bytes("[]".getBytes()));
+
     this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,
password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
             .andExpect(status().isCreated())
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
             .andExpect(jsonPath("$.index").value("broTest"))
             .andExpect(jsonPath("$.batchSize").value(1));
 
+    this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
+            .andExpect(status().isOk())
+            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+            .andExpect(content().bytes("[\"broTest\"]".getBytes()));
+
     assertEventually(() -> this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,
password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
             .andExpect(status().isOk())
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))

http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
index 9641a52..1d8e3f5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
@@ -17,10 +17,10 @@
  */
 package org.apache.metron.rest.service.impl;
 
-import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.api.DeleteBuilder;
@@ -29,12 +29,14 @@ import org.apache.curator.framework.api.GetDataBuilder;
 import org.apache.curator.framework.api.SetDataBuilder;
 import org.apache.metron.common.configuration.ConfigurationType;
 import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.common.zookeeper.ConfigurationsCache;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.service.SensorIndexingConfigService;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -44,6 +46,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -139,7 +142,85 @@ public class SensorIndexingConfigServiceImplTest {
   }
 
 
+  @Test
+  public void getAllIndicesWithOnlyParsers() throws RestException {
+    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort"));
+    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+    when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
+    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null);
+    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
+    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+
+    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+    List<String> indices = new ArrayList<String>();
+    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+    Assert.assertEquals(2, indices.size());
+    Assert.assertTrue(indices.contains("bro"));
+    Assert.assertTrue(indices.contains("snort"));
+  }
+
+  @Test
+  public void getAllIndicesWithOnlyIndexing() throws RestException {
+    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+    when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList());
+    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+    // rename bro, include snort by default configs, and disable yaf
+    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "yaf"));
+    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
+    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
+    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(false);
+
+    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+    List<String> indices = new ArrayList<String>();
+    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+    Assert.assertEquals(2, indices.size());
+    Assert.assertTrue(indices.contains("renamed_bro"));
+    Assert.assertTrue(indices.contains("snort"));
+  }
 
+  @Test
+  public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
+
+    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
+    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
+    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
+    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
+    when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
+    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
+    when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
+    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+    List<String> indices = new ArrayList<String>();
+    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+    Assert.assertEquals(3, indices.size());
+    Assert.assertTrue(indices.contains("renamed_bro"));
+    Assert.assertTrue(indices.contains("snort"));
+    Assert.assertTrue(indices.contains("yaf"));
+  }
+
+  @Test
+  public void getAllIndicesWithNoConfigs() throws RestException {
+    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+    when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList());
+    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+    when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
+
+    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+    List<String> indices = new ArrayList<String>();
+    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+    Assert.assertEquals(0, indices.size());
+  }
 
   @Test
   public void getAllTypesShouldProperlyReturnTypes() throws Exception {


Mime
View raw message