metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-995: Temporary variables in stellar enrichments which are maps do not function as expected closes apache/metron#615
Date Mon, 19 Jun 2017 14:38:06 GMT
Repository: metron
Updated Branches:
  refs/heads/master f523c1795 -> de2c871ab


METRON-995: Temporary variables in stellar enrichments which are maps do not function as expected
closes apache/metron#615


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de2c871a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de2c871a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de2c871a

Branch: refs/heads/master
Commit: de2c871abf3e259311281743952eb465362cf90c
Parents: f523c17
Author: cstella <cestella@gmail.com>
Authored: Mon Jun 19 10:37:50 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Mon Jun 19 10:37:50 2017 -0400

----------------------------------------------------------------------
 .../enrichment/handler/StellarConfig.java       | 13 ++---
 .../metron/common/stellar/StellarCompiler.java  |  2 +
 .../StellarEnrichmentConfigTest.java            | 37 +++++++++++++
 .../configuration/StellarEnrichmentTest.java    | 44 +++++++++++++++-
 .../metron/common/stellar/StellarTest.java      |  5 ++
 .../adapters/stellar/StellarAdapter.java        | 35 ++++++++++---
 .../adapters/stellar/StellarAdapterTest.java    | 55 ++++++++++++++++++++
 .../integration/EnrichmentIntegrationTest.java  |  5 ++
 .../main/config/zookeeper/enrichments/test.json |  4 +-
 9 files changed, 185 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 7d49fbd..a888b81 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -17,18 +17,18 @@
  */
 package org.apache.metron.common.configuration.enrichment.handler;
 
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
+import com.google.common.base.Joiner;
 import org.apache.metron.common.stellar.StellarAssignment;
 import org.apache.metron.common.stellar.StellarProcessor;
 import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.*;
 import java.util.function.Function;
 
 public class StellarConfig implements Config {
-
+  protected static final Logger _LOG = LoggerFactory.getLogger(StellarConfig.class);
   @Override
   public List<String> getSubgroups(Iterable<Map.Entry<String, Object>>
config) {
     boolean includeEmpty = false;
@@ -81,10 +81,10 @@ public class StellarConfig implements Config {
   {
     StellarProcessor processor = new StellarProcessor();
     List<JSONObject> messages = new ArrayList<>();
-    Map<String, String> defaultStellarStatementGroup = new HashMap<>();
+    List<String> defaultStellarStatementGroup = new ArrayList<>();
     for(Map.Entry<String, Object> kv : config) {
       if(kv.getValue() instanceof String) {
-        defaultStellarStatementGroup.put(kv.getKey(), (String)kv.getValue());
+        defaultStellarStatementGroup.add((String)kv.getValue());
       }
       else if(kv.getValue() instanceof Map) {
         JSONObject ret = new JSONObject();
@@ -103,6 +103,7 @@ public class StellarConfig implements Config {
       ret.put("", getMessage(getFields(processor, defaultStellarStatementGroup), message));
       messages.add(ret);
     }
+    _LOG.debug("Stellar enrichment split: {}", messages );
     return messages;
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
index 136728d..f4e101b 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
@@ -17,6 +17,7 @@
  */
 package org.apache.metron.common.stellar;
 
+import org.antlr.v4.runtime.ParserRuleContext;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.metron.common.dsl.Context;
 import org.apache.metron.common.dsl.Token;
@@ -347,6 +348,7 @@ public class StellarCompiler extends StellarBaseListener {
     }, DeferredFunction.class, context));
   }
 
+
   @Override
   public void exitVariable(StellarParser.VariableContext ctx) {
     final FrameContext.Context context = getArgContext();

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
index 0883672..32fb00e 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
@@ -18,7 +18,9 @@
 package org.apache.metron.common.configuration;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
 import org.apache.metron.common.configuration.enrichment.handler.Configs;
@@ -33,6 +35,41 @@ import java.util.Map;
 
 public class StellarEnrichmentConfigTest extends StellarEnrichmentTest {
 
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : [
+   "dga_model_endpoint := MAAS_GET_ENDPOINT('dga')",
+   "dga_result_map := MAAS_MODEL_APPLY( dga_model_endpoint, { 'host' : domain_without_subdomains
} )",
+   "dga_result := MAP_GET('is_malicious', dga_result_map)",
+   "is_dga := dga_result != null && dga_result == 'dga'",
+   "dga_model_version := MAP_GET('version', dga_model_endpoint)",
+   "dga_model_endpoint := null",
+   "dga_result_map := null",
+   "dga_result := null"
+        ]
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String conf;
+
+  @Test
+  public void testSplitter_listWithTemporaryVariables() throws IOException {
+    JSONObject message = new JSONObject(ImmutableMap.of("domain_without_subdomains", "yahoo.com"));
+    EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(conf, EnrichmentConfig.class);
+    Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
+    ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
+    List<JSONObject> splits = Configs.STELLAR.splitByFields(message, null, x ->
null, handler );
+    Assert.assertEquals(1, splits.size());
+    Map<String, Object> split = (Map<String, Object>)(splits.get(0)).get("");
+    Assert.assertEquals("yahoo.com", split.get("domain_without_subdomains"));
+    Assert.assertTrue(split.containsKey("dga_result"));
+    Assert.assertTrue(split.containsKey("dga_model_endpoint"));
+    Assert.assertTrue(split.containsKey("dga_result_map"));
+  }
 
   @Test
   public void testSplitter_default() throws IOException {

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
index 3af15cd..8eb9bbc 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
@@ -59,7 +59,25 @@ public class StellarEnrichmentTest {
    */
   @Multiline
   public static String defaultStellarConfig_list;
-  public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list,
defaultStellarConfig_map);
+
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : [
+          "stmt1 := TO_UPPER(source.type)",
+          "stmt4 := TO_LOWER(string)",
+          "stmt2 := TO_LOWER(stmt1)",
+          "stmt3 := TO_LOWER(string)",
+          "stmt4 := null"
+        ]
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String defaultStellarConfig_listWithTemp;
+  public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list,
defaultStellarConfig_map, defaultStellarConfig_listWithTemp);
 
 /**
    {
@@ -100,7 +118,29 @@ public class StellarEnrichmentTest {
    */
   @Multiline
   public static String groupedStellarConfig_list;
-  public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_list,
groupedStellarConfig_map);
+
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : {
+          "group1" : [
+            "stmt1 := TO_UPPER(source.type)",
+            "stmt2 := TO_LOWER(stmt1)"
+          ],
+          "group2" : [
+            "stmt3 := TO_LOWER(string)",
+            "stmt4 := TO_LOWER(string)",
+            "stmt4 := null"
+          ]
+        }
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String groupedStellarConfig_listWithTemp;
+  public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_listWithTemp,
groupedStellarConfig_list, groupedStellarConfig_map);
 
   /**
    {

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
index 98abd57..d549e19 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
@@ -183,8 +183,13 @@ public class StellarTest {
       Assert.assertEquals(ImmutableSet.of("one", "two")
                          , processor.variablesUsed("if 1 < 2 then one else two"));
     }
+    {
+      Assert.assertEquals(ImmutableSet.of("bar")
+                         , processor.variablesUsed("MAP_GET('foo', { 'foo' : bar})"));
+    }
   }
 
+
   @Test
   public void testFunctionEmptyArgs() {
     {

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
index 53ef16f..4b78821 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
@@ -32,6 +32,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -101,7 +103,9 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
     JSONObject ret = new JSONObject();
     Iterable<Map.Entry<String, Object>> stellarStatements = getStellarStatements(handler,
field);
 
+    _LOG.debug("message := {}", message);
     if(stellarStatements != null) {
+      List<String> mapEntries = new ArrayList<>();
       for (Map.Entry<String, Object> kv : stellarStatements) {
         if(kv.getKey() != null && kv.getValue() != null) {
           if (kv.getValue() instanceof String) {
@@ -114,14 +118,11 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
                 _PERF_LOG.debug("SLOW LOG: " + stellarStatement + " took" + duration + "ms");
               }
             }
+            _LOG.debug("{} := {} yields {}", kv.getKey(), stellarStatement , o);
             if (o != null && o instanceof Map) {
-              for (Map.Entry<Object, Object> valueKv : ((Map<Object, Object>)
o).entrySet()) {
-                String newKey = ((kv.getKey().length() > 0) ? kv.getKey() + "." : "")
+ valueKv.getKey();
-                message.put(newKey, valueKv.getValue());
-                ret.put(newKey, valueKv.getValue());
-              }
+              mapEntries.add(kv.getKey());
             }
-            else if(o == null) {
+            if(o == null) {
               message.remove(kv.getKey());
               ret.remove(kv.getKey());
             }
@@ -132,6 +133,28 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
           }
         }
       }
+      /*
+      We need to handle the map entries separately now.
+      We want to explode them out, so if "var" is
+      {
+        "foo" : "bar"
+      }
+      then we want "var.foo" == "bar"
+      and no "var"
+       */
+      for(String mapEntry : mapEntries) {
+        String key = mapEntry;
+        Map<Object, Object> value = (Map<Object, Object>) ret.get(key);
+        if(value != null) {
+          _LOG.debug("Exploding map: {} == {}", key, value);
+          for (Map.Entry<Object, Object> valueKv : value.entrySet()) {
+            String newKey = ((key.length() > 0) ? key + "." : "") + valueKv.getKey();
+            ret.put(newKey, valueKv.getValue());
+          }
+          //removing the map from downstream
+          ret.remove(key);
+        }
+      }
     }
     return ret;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
index 5624d4b..724221d 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
@@ -18,6 +18,7 @@
 package org.apache.metron.enrichment.adapters.stellar;
 
 import com.google.common.collect.ImmutableList;
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.StellarEnrichmentTest;
 import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
@@ -30,6 +31,8 @@ import org.json.simple.JSONObject;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.List;
+
 public class StellarAdapterTest extends StellarEnrichmentTest {
   StellarProcessor processor = new StellarProcessor();
 
@@ -131,4 +134,56 @@ public class StellarAdapterTest extends StellarEnrichmentTest {
       Assert.assertEquals(2, enriched.size());
     }
   }
+
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : {
+          "group1" : [
+            "stmt1 := TO_UPPER(source.type)",
+            "stmt2 := { 'foo' : source.type }"
+          ]
+        }
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String mapConfig_subgroup;
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : [
+            "stmt1 := TO_UPPER(source.type)",
+            "stmt2 := { 'foo' : source.type }"
+        ]
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String mapConfig_default;
+
+  private void testMapEnrichment(String config, String field) throws Exception {
+    JSONObject message = getMessage();
+    EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class);
+    Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
+    ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
+    JSONObject enriched = enrich(message, field, handler);
+    Assert.assertEquals(2, enriched.size());
+    Assert.assertEquals("stellar_test", enriched.get("stmt2.foo"));
+    Assert.assertEquals("stellar_test".toUpperCase(), enriched.get("stmt1"));
+  }
+
+  @Test
+  public void testMapEnrichment_subgroup() throws Exception {
+    testMapEnrichment(mapConfig_subgroup, "group1");
+  }
+
+  @Test
+  public void testMapEnrichment_default() throws Exception {
+    testMapEnrichment(mapConfig_default, "");
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 9634805..2149cb9 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -260,6 +260,11 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
     Assert.assertNotNull(jsonDoc.get(DST_IP));
 
     Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
+    Assert.assertNotNull(jsonDoc.get("map.blah"));
+    Assert.assertNull(jsonDoc.get("map"));
+    Assert.assertNotNull(jsonDoc.get("one"));
+    Assert.assertEquals(1, jsonDoc.get("one"));
+    Assert.assertEquals(1, jsonDoc.get("map.blah"));
     Assert.assertNotNull(jsonDoc.get("foo"));
     Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS"));
     Assert.assertNotNull(jsonDoc.get("bar"));

http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
index 1037b69..bba6679 100644
--- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
@@ -16,7 +16,9 @@
       "stellar" : {
         "config" : {
           "numeric" : {
-                      "foo": "1 + 1"
+                      "map" : "{ 'blah' : 1}"
+                      ,"one" : "MAP_GET('blah', map)"
+                      ,"foo": "1 + 1"
                       }
           ,"ALL_CAPS" : "TO_UPPER(source.type)"
           ,"src_enrichment" : {


Mime
View raw message