From commits-return-3385-archive-asf-public=cust-asf.ponee.io@metron.apache.org Fri Jul 6 17:42:27 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id DB618180627 for ; Fri, 6 Jul 2018 17:42:26 +0200 (CEST) Received: (qmail 21264 invoked by uid 500); 6 Jul 2018 15:42:26 -0000 Mailing-List: contact commits-help@metron.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.apache.org Delivered-To: mailing list commits@metron.apache.org Received: (qmail 21255 invoked by uid 99); 6 Jul 2018 15:42:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Jul 2018 15:42:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7B108E08ED; Fri, 6 Jul 2018 15:42:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cestella@apache.org To: commits@metron.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: metron git commit: METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083 Date: Fri, 6 Jul 2018 15:42:25 +0000 (UTC) Repository: metron Updated Branches: refs/heads/master 4eb9a8ebc -> f183d987a METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f183d987 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f183d987 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f183d987 Branch: refs/heads/master Commit: f183d987ab1c9856c585c45b0f9f549aadccb65f Parents: 4eb9a8e Author: cstella Authored: Fri Jul 6 11:40:54 2018 -0400 Committer: cstella Committed: Fri Jul 6 11:40:54 2018 -0400 ---------------------------------------------------------------------- .../transformation/FieldTransformations.java | 1 + .../RegexSelectTransformation.java | 97 +++++++++++ .../RegexSelectTransformationTest.java | 169 +++++++++++++++++++ metron-platform/metron-parsers/README.md | 23 +++ 4 files changed, 290 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java index 95ff390..d80c1a4 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java @@ -26,6 +26,7 @@ public enum FieldTransformations { ,STELLAR(new StellarTransformation()) ,SELECT(new SelectTransformation()) ,RENAME(new RenameTransformation()) + ,REGEX_SELECT(new RegexSelectTransformation()) ; FieldTransformation mapping; FieldTransformations(FieldTransformation mapping) { http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java new file mode 100644 index 0000000..b8d1c07 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.field.transformation; + +import com.google.common.collect.Iterables; +import org.apache.metron.stellar.common.utils.PatternCache; +import org.apache.metron.stellar.dsl.Context; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public class RegexSelectTransformation implements FieldTransformation { + @Override + public Map map( Map input + , List outputField + , LinkedHashMap fieldMappingConfig + , Context context + , Map... sensorConfig + ) { + String outField = null; + if(!(outputField == null || outputField.isEmpty())) { + outField = outputField.get(0); + } + String inVal = null; + if(!(input == null || input.isEmpty() || input.size() > 1)) { + Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue(); + if(inValObj != null) { + inVal = inValObj.toString(); + } + } + Map ret = new HashMap<>(1); + if(outField == null || inVal == null) { + //in the situation where we don't have valid input or output, then we want to do nothing + return ret; + } + for(Map.Entry valToRegex : fieldMappingConfig.entrySet()) { + if(isMatch(valToRegex.getValue(), inVal)) { + ret.put(outField, valToRegex.getKey()); + break; + } + } + return ret; + } + + /** + * Return true if there is a regex match or false otherwise. + * @param regexes This could be either a list of regexes or a single regex + * @param field The field to match + * @return + */ + private static boolean isMatch(Object regexes, String field) { + if(regexes instanceof String) { + return isMatch((String)regexes, field); + } + else if(regexes instanceof List) { + for(Object regex : (List)regexes) { + if(isMatch(regex.toString(), field)) { + return true; + } + } + } + return false; + } + + private static boolean isMatch(String regex, String field) { + try { + Pattern p = PatternCache.INSTANCE.getPattern(regex); + if (p == null) { + return false; + } + return p.asPredicate().test(field); + } + catch(PatternSyntaxException pse) { + return false; + } + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java new file mode 100644 index 0000000..8d9e935 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.field.transformation; + +import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.configuration.FieldTransformer; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class RegexSelectTransformationTest { + /** + { + "fieldTransformations" : [ + { + "input" : "in_field" + , "output" : "out_field" + , "transformation" : "REGEX_SELECT" + , "config" : { + "option_1" : ".*foo.*", + "option_2" : [ ".*metron.*", ".*mortron.*" ] + } + } + ] + } + */ + @Multiline + public static String routeSingleInSingleOut; + + private String transform(String in, String config) throws Exception { + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); + FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap() {{ + put("in_field", in); + put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message + }}); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + return (String) input.get("out_field"); + } + + @Test + public void smoketest() throws Exception{ + Assert.assertEquals("option_1", transform("foo", routeSingleInSingleOut)); + Assert.assertNull(transform("bar", routeSingleInSingleOut)); + } + + @Test + public void testListOfRegexes() throws Exception { + Assert.assertEquals("option_2", transform("I am mortron", routeSingleInSingleOut)); + Assert.assertEquals("option_2", transform("metron is for smelling", routeSingleInSingleOut)); + } + @Test + public void testPrecedence() throws Exception { + Assert.assertEquals("option_1", transform("metron is for foorensic cybersecurity", routeSingleInSingleOut)); + } + + /** + { + "fieldTransformations" : [ + { + "output" : "out_field" + , "transformation" : "REGEX_SELECT" + , "config" : { + "option_1" : ".*foo.*", + "option_2" : [ ".*metron.*", ".*mortron.*" ] + } + } + ] + } + */ + @Multiline + public static String routeMissingInput; + + @Test + public void testMissingInput() throws Exception { + Assert.assertNull(transform("metron", routeMissingInput)); + } + + /** + { + "fieldTransformations" : [ + { + "input" : "in_field" + , "transformation" : "REGEX_SELECT" + , "config" : { + "option_1" : ".*foo.*", + "option_2" : [ ".*metron.*", ".*mortron.*" ] + } + } + ] + } + */ + @Multiline + public static String routeMissingOutput; + + @Test + public void testMissingOutput() throws Exception { + Assert.assertNull(transform("metron", routeMissingOutput)); + } + + /** + { + "fieldTransformations" : [ + { + "input" : "in_field" + ,"output" : [ "out_field", "baz_field" ] + , "transformation" : "REGEX_SELECT" + , "config" : { + "option_1" : ".*foo.*", + "option_2" : [ ".*metron.*", ".*mortron.*" ] + } + } + ] + } + */ + @Multiline + public static String routeMultiOutput; + + @Test + public void testMultiOutput() throws Exception{ + Assert.assertEquals("option_1", transform("foo", routeMultiOutput)); + Assert.assertNull(transform("bar", routeMultiOutput)); + } + + /** + { + "fieldTransformations" : [ + { + "input" : "in_field" + ,"output" : "out_field" + , "transformation" : "REGEX_SELECT" + , "config" : { + "option_1" : "[a-z", + "option_2" : [ ".*metron.*", ".*mortron.*" ] + } + } + ] + } + */ + @Multiline + public static String routeBadRegex; + + @Test + public void testBadRegex() throws Exception{ + Assert.assertEquals("option_2", transform("metron", routeBadRegex)); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 7ddfdea..c453758 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -338,6 +338,29 @@ The following config will rename the fields `old_field` and `different_old_field ] } ``` +* `REGEX_SELECT` : This transformation lets users set an output field to one of a set of possibilities based on matching regexes. This transformation is useful when the number or conditions are large enough to make a stellar language match statement unwieldy. + +The following config will set the field `logical_source_type` to one of the +following, dependent upon the value of the `pix_type` field: +* `cisco-6-302` if `pix_type` starts with either `6-302` or `06-302` +* `cisco-5-304` if `pix_type` starts with `5-304` +``` +{ +... + "fieldTransformations" : [ + { + "transformation" : "REGEX_ROUTING" + ,"input" : "pix_type" + ,"output" : "logical_source_type" + ,"config" : { + "cisco-6-302" : [ "^6-302.*", "^06-302.*"] + "cisco-5-304" : "^5-304.*" + } + } + ] +... +} +``` ### Assignment to `null`