metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-401 ParseException Thrown by ThreatIntelJoinBolt (nickwallen via cestella) closes apache/incubator-metron#239
Date Thu, 01 Sep 2016 10:56:19 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master b7e16d2f9 -> 839bd2093


METRON-401 ParseException Thrown by ThreatIntelJoinBolt (nickwallen via cestella) closes apache/incubator-metron#239


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/839bd209
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/839bd209
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/839bd209

Branch: refs/heads/master
Commit: 839bd20931d62219f4e375199601899ab3f71001
Parents: b7e16d2
Author: nickwallen <nick@nickallen.org>
Authored: Thu Sep 1 20:56:07 2016 +1000
Committer: cstella <cestella@gmail.com>
Committed: Thu Sep 1 20:56:07 2016 +1000

----------------------------------------------------------------------
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |   5 +-
 .../bolt/ThreatIntelJoinBoltTest.java           | 138 ++++++----
 .../threatintel/triage/ThreatTriageTest.java    | 253 +++++++++++--------
 3 files changed, 230 insertions(+), 166 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/839bd209/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 00d0498..9506697 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -40,10 +40,10 @@ import java.util.Map;
 
 public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
 
-  protected static final Logger LOG = LoggerFactory
-          .getLogger(ThreatIntelJoinBolt.class);
+  protected static final Logger LOG = LoggerFactory.getLogger(ThreatIntelJoinBolt.class);
   private FunctionResolver functionResolver;
   private org.apache.metron.common.dsl.Context stellarContext;
+
   public ThreatIntelJoinBolt(String zookeeperUrl) {
     super(zookeeperUrl);
   }
@@ -73,6 +73,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
     this.stellarContext = new Context.Builder()
                                 .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
                                 .build();
+    this.functionResolver = StellarFunctions.FUNCTION_RESOLVER();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/839bd209/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
index b3004bb..bb539f3 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.enrichment.bolt;
 
 import com.fasterxml.jackson.databind.JsonMappingException;
 import junit.framework.Assert;
+import junit.framework.TestCase;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
@@ -39,33 +40,33 @@ import java.util.Map;
 public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
 
   /**
-   {
-   "field1": "value1",
-   "enrichedField1": "enrichedValue1",
-   "source.type": "test"
-   }
+   * {
+   * "field1": "value1",
+   * "enrichedField1": "enrichedValue1",
+   * "source.type": "test"
+   * }
    */
   @Multiline
   private String messageString;
 
   /**
-   {
-   "field1": "value1",
-   "enrichedField1": "enrichedValue1",
-   "source.type": "test",
-   "threatintels.field.end.ts": "timing"
-   }
+   * {
+   * "field1": "value1",
+   * "enrichedField1": "enrichedValue1",
+   * "source.type": "test",
+   * "threatintels.field.end.ts": "timing"
+   * }
    */
   @Multiline
   private String messageWithTimingString;
 
   /**
-   {
-   "field1": "value1",
-   "enrichedField1": "enrichedValue1",
-   "source.type": "test",
-   "threatintels.field": "threatIntelValue"
-   }
+   * {
+   * "field1": "value1",
+   * "enrichedField1": "enrichedValue1",
+   * "source.type": "test",
+   * "threatintels.field": "threatIntelValue"
+   * }
    */
   @Multiline
   private String alertMessageString;
@@ -83,31 +84,75 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
   }
 
   /**
-    {
-    "riskLevelRules" : {
-        "enrichedField1 == 'enrichedValue1'" : 10
-                          }
-   ,"aggregator" : "MAX"
-   }
+   * {
+   *  "riskLevelRules" : {
+   *    "enrichedField1 == 'enrichedValue1'" : 10
+   *  },
+   *  "aggregator" : "MAX"
+   * }
    */
   @Multiline
-  private static String threatTriageConfigStr;
+  private static String testWithTriageConfig;
+
+  @Test
+  public void testWithTriage() throws IOException {
+    test(testWithTriageConfig, false);
+  }
+
+  /**
+   * {
+   *  "riskLevelRules" : {
+   *    "enrichedField1 == 'enrichedValue1": 10
+   *  },
+   *  "aggregator" : "MAX"
+   * }
+   */
+  @Multiline
+  private static String testWithBadTriageRuleConfig;
+
+  @Test
+  public void testWithBadTriageRule() throws IOException {
+    test(testWithBadTriageRuleConfig, true);
+  }
+
+  @Test
+  public void testWithoutTriage() throws IOException {
+    test(null, false);
+  }
+
+  /**
+   * {
+   *   "riskLevelRules": {
+   *      "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))": 10
+   *   },
+   *   "aggregator": "MAX"
+   * }
+   */
+  @Multiline
+  private static String testWithStellarFunctionConfig;
+
+  @Test
+  public void testWithStellarFunction() throws IOException {
+    test(testWithStellarFunctionConfig, false);
+  }
 
   public void test(String threatTriageConfig, boolean badConfig) throws IOException {
+
     ThreatIntelJoinBolt threatIntelJoinBolt = new ThreatIntelJoinBolt("zookeeperUrl");
     threatIntelJoinBolt.setCuratorFramework(client);
     threatIntelJoinBolt.setTreeCache(cache);
-    SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(new FileInputStream(sampleSensorEnrichmentConfigPath),
SensorEnrichmentConfig.class);
+
+    SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(
+            new FileInputStream(sampleSensorEnrichmentConfigPath), SensorEnrichmentConfig.class);
     boolean withThreatTriage = threatTriageConfig != null;
-    if(withThreatTriage) {
+    if (withThreatTriage) {
       try {
         enrichmentConfig.getThreatIntel().setTriageConfig(JSONUtils.INSTANCE.load(threatTriageConfig,
ThreatTriageConfig.class));
-        if(badConfig) {
+        if (badConfig) {
           Assert.fail(threatTriageConfig + "\nThis should not parse!");
         }
-      }
-      catch(JsonMappingException pe) {
-        if(!badConfig) {
+      } catch (JsonMappingException pe) {
+        if (!badConfig) {
           throw pe;
         }
       }
@@ -116,49 +161,32 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest
{
     threatIntelJoinBolt.withMaxCacheSize(100);
     threatIntelJoinBolt.withMaxTimeRetain(10000);
     threatIntelJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
+
     Map<String, Object> fieldMap = threatIntelJoinBolt.getFieldMap("incorrectSourceType");
     Assert.assertNull(fieldMap);
+
     fieldMap = threatIntelJoinBolt.getFieldMap(sensorType);
     Assert.assertTrue(fieldMap.containsKey("hbaseThreatIntel"));
+
     Map<String, JSONObject> streamMessageMap = new HashMap<>();
     streamMessageMap.put("message", message);
     JSONObject joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap);
     Assert.assertFalse(joinedMessage.containsKey("is_alert"));
+
     streamMessageMap.put("message", messageWithTiming);
     joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap);
     Assert.assertFalse(joinedMessage.containsKey("is_alert"));
+
     streamMessageMap.put("message", alertMessage);
     joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap);
     Assert.assertTrue(joinedMessage.containsKey("is_alert") && "true".equals(joinedMessage.get("is_alert")));
+
     if(withThreatTriage && !badConfig) {
-      Assert.assertTrue(joinedMessage.containsKey("threat.triage.level") && Math.abs(10d
- (Double) joinedMessage.get("threat.triage.level")) < 1e-10);
+      Assert.assertTrue(joinedMessage.containsKey("threat.triage.level") &&
+              Math.abs(10d - (Double) joinedMessage.get("threat.triage.level")) < 1e-10);
     }
     else {
       Assert.assertFalse(joinedMessage.containsKey("threat.triage.level"));
     }
   }
-  /**
-    {
-    "riskLevelRules" : {
-        "enrichedField1 == 'enrichedValue1" : 10
-                          }
-   ,"aggregator" : "MAX"
-   }
-   */
-  @Multiline
-  private static String badRuleThreatTriageConfigStr;
-
-
-  @Test
-  public void testWithTriage() throws IOException {
-    test(threatTriageConfigStr, false);
-  }
-  @Test
-  public void testWithBadTriageRule() throws IOException {
-    test(badRuleThreatTriageConfigStr, true);
-  }
-  @Test
-  public void testWithoutTriage() throws IOException {
-    test(null, false);
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/839bd209/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
index 37292e2..92191b6 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
@@ -20,7 +20,6 @@ package org.apache.metron.threatintel.triage;
 
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
 import org.apache.metron.common.dsl.Context;
 import org.apache.metron.common.dsl.StellarFunctions;
 import org.apache.metron.common.utils.JSONUtils;
@@ -32,96 +31,99 @@ import java.util.HashMap;
 
 public class ThreatTriageTest {
   /**
-   {
-    "threatIntel" : {
-      "triageConfig" :
-      {
-        "riskLevelRules" : {
-            "user.type in [ 'admin', 'power' ] and asset.type == 'web'" : 10
-           ,"asset.type == 'web'" : 5
-          ,"user.type == 'normal'  and asset.type == 'web'" : 0
-          ,"user.type in whitelist" : -1
-                          }
-        ,"aggregator" : "MAX"
-      },
-      "config" : {
-        "whitelist" : [ "abnormal" ]
-                 }
-    }
-   }
+   * {
+   *  "threatIntel": {
+   *    "triageConfig": {
+   *      "riskLevelRules" : {
+   *        "user.type in [ 'admin', 'power' ] and asset.type == 'web'" : 10,
+   *        "asset.type == 'web'" : 5,
+   *        "user.type == 'normal'  and asset.type == 'web'" : 0,
+   *        "user.type in whitelist" : -1
+   *      },
+   *      "aggregator" : "MAX"
+   *    },
+   *    "config": {
+   *      "whitelist": [ "abnormal" ]
+   *    }
+   *  }
+   * }
    */
   @Multiline
   public static String smokeTestProcessorConfig;
 
-  private static ThreatTriageProcessor getProcessor(String config) throws IOException {
-    SensorEnrichmentConfig c = JSONUtils.INSTANCE.load(config, SensorEnrichmentConfig.class);
-    return new ThreatTriageProcessor(c, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
-  }
-
   @Test
   public void smokeTest() throws Exception {
     ThreatTriageProcessor threatTriageProcessor = getProcessor(smokeTestProcessorConfig);
-    Assert.assertEquals("Expected a score of 0"
-                       , 0d
-                       ,new ThreatTriageProcessor(new SensorEnrichmentConfig(), StellarFunctions.FUNCTION_RESOLVER(),
Context.EMPTY_CONTEXT()).apply(new HashMap<Object, Object>() {{
-                          put("user.type", "admin");
-                          put("asset.type", "web");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-    Assert.assertEquals("Expected a score of 10"
-                       , 10d
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "admin");
-                          put("asset.type", "web");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-    Assert.assertEquals("Expected a score of 5"
-                       , 5d
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "normal");
-                          put("asset.type", "web");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-    Assert.assertEquals("Expected a score of 0"
-                       , 0d
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "foo");
-                          put("asset.type", "bar");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-    Assert.assertEquals("Expected a score of -Inf"
-                       , Double.NEGATIVE_INFINITY
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "abnormal");
-                          put("asset.type", "bar");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
+
+    Assert.assertEquals(
+            "Expected a score of 0",
+            0d,
+            new ThreatTriageProcessor(
+                    new SensorEnrichmentConfig(),
+                    StellarFunctions.FUNCTION_RESOLVER(),
+                    Context.EMPTY_CONTEXT()).apply(
+                            new HashMap<Object, Object>() {{
+                              put("user.type", "admin");
+                              put("asset.type", "web");
+                            }}),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of 10",
+            10d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "admin");
+                      put("asset.type", "web");
+                    }}
+            ),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of 5",
+            5d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "normal");
+                      put("asset.type", "web");
+                    }}
+            ),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of 0",
+            0d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "foo");
+                      put("asset.type", "bar");
+                    }}),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of -Inf",
+            Double.NEGATIVE_INFINITY,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "abnormal");
+                      put("asset.type", "bar");
+                    }}),
+            1e-10);
   }
 
   /**
-   {
-    "threatIntel" : {
-      "triageConfig" :
-      {
-        "riskLevelRules" : {
-            "user.type in [ 'admin', 'power' ] and asset.type == 'web'" : 10
-           ,"asset.type == 'web'" : 5
-           ,"user.type == 'normal'  and asset.type == 'web'" : 0
-                          }
-        ,"aggregator" : "POSITIVE_MEAN"
-      }
-                   }
-   }
+   * {
+   *  "threatIntel": {
+   *  "triageConfig": {
+   *    "riskLevelRules" : {
+   *      "user.type in [ 'admin', 'power' ] and asset.type == 'web'" : 10,
+   *      "asset.type == 'web'" : 5,
+   *      "user.type == 'normal' and asset.type == 'web'" : 0
+   *     },
+   *     "aggregator" : "POSITIVE_MEAN"
+   *    }
+   *  }
+   * }
    */
   @Multiline
   public static String positiveMeanProcessorConfig;
@@ -130,33 +132,66 @@ public class ThreatTriageTest {
   public void positiveMeanAggregationTest() throws Exception {
 
     ThreatTriageProcessor threatTriageProcessor = getProcessor(positiveMeanProcessorConfig);
-    Assert.assertEquals("Expected a score of 0"
-                       , 5d
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "normal");
-                          put("asset.type", "web");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-    Assert.assertEquals("Expected a score of 7.5"
-                       , (10 + 5)/2.0
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "admin");
-                          put("asset.type", "web");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
-
-    Assert.assertEquals("Expected a score of 0"
-                       , 0d
-                       , threatTriageProcessor.apply(new HashMap<Object, Object>()
{{
-                          put("user.type", "foo");
-                          put("asset.type", "bar");
-                                        }}
-                                        )
-                       , 1e-10
-                       );
+    Assert.assertEquals(
+            "Expected a score of 0",
+            5d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "normal");
+                      put("asset.type", "web");
+                    }}),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of 7.5",
+            (10 + 5)/2.0,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "admin");
+                      put("asset.type", "web");
+                    }}),
+            1e-10);
+
+    Assert.assertEquals(
+            "Expected a score of 0",
+            0d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("user.type", "foo");
+                      put("asset.type", "bar");
+                    }}),
+            1e-10);
+  }
+
+  /**
+   * {
+   *    "threatIntel" : {
+   *      "triageConfig": {
+   *        "riskLevelRules": {
+   *          "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))" : 10
+   *        },
+   *        "aggregator" : "MAX"
+   *      }
+   *    }
+   * }
+   */
+  @Multiline
+  private static String testWithStellarFunction;
+
+  @Test
+  public void testWithStellarFunction() throws Exception {
+    ThreatTriageProcessor threatTriageProcessor = getProcessor(testWithStellarFunction);
+    Assert.assertEquals(
+            10d,
+            threatTriageProcessor.apply(
+                    new HashMap<Object, Object>() {{
+                      put("ip_dst_addr", "172.2.2.2");
+                    }}),
+            1e-10);
+  }
+
+  private static ThreatTriageProcessor getProcessor(String config) throws IOException {
+    SensorEnrichmentConfig c = JSONUtils.INSTANCE.load(config, SensorEnrichmentConfig.class);
+    return new ThreatTriageProcessor(c, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
   }
 }


Mime
View raw message