metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [35/52] [abbrv] metron git commit: METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976
Date Wed, 18 Apr 2018 15:00:05 GMT
METRON-1504: Enriching missing values does not match the semantics between the new enrichment
topology and old closes apache/incubator-metron#976


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4
Parents: 46bc63d
Author: cstella <cestella@gmail.com>
Authored: Mon Apr 9 11:47:51 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Mon Apr 9 11:47:51 2018 -0400

----------------------------------------------------------------------
 .../enrichment/parallel/ParallelEnricher.java   |  3 +
 .../integration/EnrichmentIntegrationTest.java  | 12 +++-
 .../parallel/ParallelEnricherTest.java          | 59 +++++++++++++++++++-
 3 files changed, 70 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 6ddb892..b10c148 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -170,6 +170,9 @@ public class ParallelEnricher {
         for(Object o : m.keySet()) {
           String field = (String) o;
           Object value = m.get(o);
+          if(value == null) {
+            continue;
+          }
           CacheKey cacheKey = new CacheKey(field, value, config);
           String prefix = adapter.getOutputPrefix(cacheKey);
           Supplier<JSONObject> supplier = () -> {

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 267ca62..3c55c95 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
 
   private static List<byte[]> getInputMessages(String path){
     try{
-      return TestUtils.readSampleData(path);
+      List<byte[]> ret = TestUtils.readSampleData(path);
+      {
+        //we want one of the fields without a destination IP to ensure that enrichments can
function
+        Map<String, Object> sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size()
-1))
+                                                                       , JSONUtils.MAP_SUPPLIER);
+        sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName());
+        ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp));
+      }
+      return ret;
     }catch(IOException ioe){
       return null;
     }
@@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
 
     //ensure we always have a source ip and destination ip
     Assert.assertNotNull(jsonDoc.get(SRC_IP));
-    Assert.assertNotNull(jsonDoc.get(DST_IP));
-
     Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
     Assert.assertNotNull(jsonDoc.get("map.blah"));
     Assert.assertNull(jsonDoc.get("map"));

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index 4a4573b..d4fcdf4 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -79,7 +79,39 @@ public class ParallelEnricherTest {
       }
     }.ofType("ENRICHMENT");
     adapter.initializeAdapter(new HashMap<>());
-    enrichmentsByType = ImmutableMap.of("stellar", adapter);
+    EnrichmentAdapter<CacheKey> dummy = new EnrichmentAdapter<CacheKey>() {
+      @Override
+      public void logAccess(CacheKey value) {
+
+      }
+
+      @Override
+      public JSONObject enrich(CacheKey value) {
+        return null;
+      }
+
+      @Override
+      public boolean initializeAdapter(Map<String, Object> config) {
+        return false;
+      }
+
+      @Override
+      public void updateAdapter(Map<String, Object> config) {
+
+      }
+
+      @Override
+      public void cleanup() {
+
+      }
+
+      @Override
+      public String getOutputPrefix(CacheKey value) {
+        return null;
+      }
+    };
+
+    enrichmentsByType = ImmutableMap.of("stellar", adapter, "dummy", dummy);
     enricher = new ParallelEnricher(enrichmentsByType, infrastructure, false);
   }
 
@@ -115,6 +147,31 @@ public class ParallelEnricherTest {
     Assert.assertEquals("TEST", ret.get("ALL_CAPS"));
     Assert.assertEquals(0, result.getEnrichmentErrors().size());
   }
+/**
+   * {
+  "enrichment": {
+    "fieldMap": {
+      "dummy" : ["notthere"]
+    }
+  ,"fieldToTypeMap": { }
+  },
+  "threatIntel": { }
+}
+   */
+  @Multiline
+  public static String nullConfig;
+
+  @Test
+  public void testNullEnrichment() throws Exception {
+    SensorEnrichmentConfig config = JSONUtils.INSTANCE.load(nullConfig, SensorEnrichmentConfig.class);
+    config.getConfiguration().putIfAbsent("stellarContext", stellarContext);
+    JSONObject message = new JSONObject() {{
+      put(Constants.SENSOR_TYPE, "test");
+    }};
+    ParallelEnricher.EnrichmentResult result = enricher.apply(message, EnrichmentStrategies.ENRICHMENT,
config, null);
+    JSONObject ret = result.getResult();
+    Assert.assertEquals("Got the wrong result count: " + ret, 4, ret.size());
+  }
 
   /**
    * {


Mime
View raw message