metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083
Date Fri, 06 Jul 2018 15:42:25 GMT
Repository: metron
Updated Branches:
  refs/heads/master 4eb9a8ebc -> f183d987a


METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f183d987
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f183d987
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f183d987

Branch: refs/heads/master
Commit: f183d987ab1c9856c585c45b0f9f549aadccb65f
Parents: 4eb9a8e
Author: cstella <cestella@gmail.com>
Authored: Fri Jul 6 11:40:54 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Fri Jul 6 11:40:54 2018 -0400

----------------------------------------------------------------------
 .../transformation/FieldTransformations.java    |   1 +
 .../RegexSelectTransformation.java              |  97 +++++++++++
 .../RegexSelectTransformationTest.java          | 169 +++++++++++++++++++
 metron-platform/metron-parsers/README.md        |  23 +++
 4 files changed, 290 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index 95ff390..d80c1a4 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -26,6 +26,7 @@ public enum FieldTransformations {
   ,STELLAR(new StellarTransformation())
   ,SELECT(new SelectTransformation())
   ,RENAME(new RenameTransformation())
+  ,REGEX_SELECT(new RegexSelectTransformation())
   ;
   FieldTransformation mapping;
   FieldTransformations(FieldTransformation mapping) {

http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
new file mode 100644
index 0000000..b8d1c07
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.apache.metron.stellar.common.utils.PatternCache;
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class RegexSelectTransformation implements FieldTransformation {
+  @Override
+  public Map<String, Object> map( Map<String, Object> input
+                                , List<String> outputField
+                                , LinkedHashMap<String, Object> fieldMappingConfig
+                                , Context context
+                                , Map<String, Object>... sensorConfig
+                                ) {
+    String outField = null;
+    if(!(outputField == null || outputField.isEmpty())) {
+      outField = outputField.get(0);
+    }
+    String inVal = null;
+    if(!(input == null || input.isEmpty() || input.size() > 1)) {
+      Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue();
+      if(inValObj != null) {
+        inVal = inValObj.toString();
+      }
+    }
+    Map<String, Object> ret = new HashMap<>(1);
+    if(outField == null || inVal == null) {
+      //in the situation where we don't have valid input or output, then we want to do nothing
+      return ret;
+    }
+    for(Map.Entry<String, Object> valToRegex : fieldMappingConfig.entrySet()) {
+      if(isMatch(valToRegex.getValue(), inVal)) {
+        ret.put(outField, valToRegex.getKey());
+        break;
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * Return true if there is a regex match or false otherwise.
+   * @param regexes This could be either a list of regexes or a single regex
+   * @param field The field to match
+   * @return
+   */
+  private static boolean isMatch(Object regexes, String field) {
+    if(regexes instanceof String) {
+      return isMatch((String)regexes, field);
+    }
+    else if(regexes instanceof List) {
+      for(Object regex : (List)regexes) {
+        if(isMatch(regex.toString(), field)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  private static boolean isMatch(String regex, String field) {
+    try {
+      Pattern p = PatternCache.INSTANCE.getPattern(regex);
+      if (p == null) {
+        return false;
+      }
+      return p.asPredicate().test(field);
+    }
+    catch(PatternSyntaxException pse) {
+      return false;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
new file mode 100644
index 0000000..8d9e935
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class RegexSelectTransformationTest {
+  /**
+   {
+    "fieldTransformations" : [
+          {
+            "input" : "in_field"
+          , "output" : "out_field"
+          , "transformation" : "REGEX_SELECT"
+          , "config" : {
+              "option_1" : ".*foo.*",
+              "option_2" : [ ".*metron.*", ".*mortron.*" ]
+             }
+          }
+                      ]
+   }
+   */
+  @Multiline
+  public static String routeSingleInSingleOut;
+
+  private String transform(String in, String config) throws Exception {
+    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config));
+    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+    JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+      put("in_field", in);
+      put("dummy_field", "dummy"); //this is added to ensure that it looks like something
approaching a real message
+    }});
+    handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+    return (String) input.get("out_field");
+  }
+
+  @Test
+  public void smoketest() throws Exception{
+    Assert.assertEquals("option_1", transform("foo", routeSingleInSingleOut));
+    Assert.assertNull(transform("bar", routeSingleInSingleOut));
+  }
+
+  @Test
+  public void testListOfRegexes() throws Exception {
+    Assert.assertEquals("option_2", transform("I am mortron", routeSingleInSingleOut));
+    Assert.assertEquals("option_2", transform("metron is for smelling", routeSingleInSingleOut));
+  }
+  @Test
+  public void testPrecedence() throws Exception {
+    Assert.assertEquals("option_1", transform("metron is for foorensic cybersecurity", routeSingleInSingleOut));
+  }
+
+  /**
+   {
+    "fieldTransformations" : [
+          {
+           "output" : "out_field"
+          , "transformation" : "REGEX_SELECT"
+          , "config" : {
+              "option_1" : ".*foo.*",
+              "option_2" : [ ".*metron.*", ".*mortron.*" ]
+             }
+          }
+                      ]
+   }
+   */
+  @Multiline
+  public static String routeMissingInput;
+
+  @Test
+  public void testMissingInput() throws Exception {
+    Assert.assertNull(transform("metron", routeMissingInput));
+  }
+
+  /**
+   {
+    "fieldTransformations" : [
+          {
+           "input" : "in_field"
+          , "transformation" : "REGEX_SELECT"
+          , "config" : {
+              "option_1" : ".*foo.*",
+              "option_2" : [ ".*metron.*", ".*mortron.*" ]
+             }
+          }
+                      ]
+   }
+   */
+  @Multiline
+  public static String routeMissingOutput;
+
+  @Test
+  public void testMissingOutput() throws Exception {
+    Assert.assertNull(transform("metron", routeMissingOutput));
+  }
+
+  /**
+   {
+    "fieldTransformations" : [
+          {
+           "input" : "in_field"
+          ,"output" : [ "out_field", "baz_field" ]
+          , "transformation" : "REGEX_SELECT"
+          , "config" : {
+              "option_1" : ".*foo.*",
+              "option_2" : [ ".*metron.*", ".*mortron.*" ]
+             }
+          }
+                      ]
+   }
+   */
+  @Multiline
+  public static String routeMultiOutput;
+
+  @Test
+  public void testMultiOutput() throws Exception{
+    Assert.assertEquals("option_1", transform("foo", routeMultiOutput));
+    Assert.assertNull(transform("bar", routeMultiOutput));
+  }
+
+  /**
+   {
+    "fieldTransformations" : [
+          {
+           "input" : "in_field"
+          ,"output" : "out_field"
+          , "transformation" : "REGEX_SELECT"
+          , "config" : {
+              "option_1" : "[a-z",
+              "option_2" : [ ".*metron.*", ".*mortron.*" ]
+             }
+          }
+                      ]
+   }
+   */
+  @Multiline
+  public static String routeBadRegex;
+
+  @Test
+  public void testBadRegex() throws Exception{
+    Assert.assertEquals("option_2", transform("metron", routeBadRegex));
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index 7ddfdea..c453758 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -338,6 +338,29 @@ The following config will rename the fields `old_field` and `different_old_field
                       ]
 }
 ```
+* `REGEX_SELECT` : This transformation lets users set an output field to one of a set of
possibilities based on matching regexes. This transformation is useful when the number or
conditions are large enough to make a stellar language match statement unwieldy.
+ 
+The following config will set the field `logical_source_type` to one of the
+following, dependent upon the value of the `pix_type` field:
+* `cisco-6-302` if `pix_type` starts with either `6-302` or `06-302`
+* `cisco-5-304` if `pix_type` starts with `5-304`
+```
+{
+...
+  "fieldTransformations" : [
+    {
+     "transformation" : "REGEX_ROUTING"
+    ,"input" :  "pix_type"
+    ,"output" :  "logical_source_type"
+    ,"config" : {
+      "cisco-6-302" : [ "^6-302.*", "^06-302.*"]
+      "cisco-5-304" : "^5-304.*"
+                }
+    }
+                           ]
+...  
+}
+```
 
 
 ### Assignment to `null`


Mime
View raw message