metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [04/11] metron git commit: METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002
Date Mon, 14 May 2018 19:28:22 GMT
METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b4f0b84
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b4f0b84
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b4f0b84

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 2b4f0b84062d65f9400421d66ec3b7d6d093bebf
Parents: 1c5435c
Author: cstella <cestella@gmail.com>
Authored: Wed Apr 25 11:49:56 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Wed Apr 25 11:49:56 2018 -0400

----------------------------------------------------------------------
 .../common/configuration/FieldTransformer.java  |  4 +-
 .../transformation/FieldTransformations.java    |  1 +
 .../transformation/RenameTransformation.java    | 55 +++++++++++
 .../transformation/FieldTransformationTest.java | 17 +---
 .../RenameTransformationTest.java               | 99 ++++++++++++++++++++
 metron-platform/metron-parsers/README.md        | 25 ++++-
 6 files changed, 183 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
index df80691..43ce9d8 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
@@ -94,7 +94,9 @@ public class FieldTransformer implements Serializable {
 
       if (output == null || output.isEmpty()) {
         if (input == null || input.isEmpty()) {
-          throw new IllegalStateException("You must specify an input field if you want to
leave the output fields empty");
+          //both are empty, so let's set them both to null
+          output = null;
+          input = null;
         } else {
           output = input;
         }

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index a905123..95ff390 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -25,6 +25,7 @@ public enum FieldTransformations {
   ,REMOVE(new RemoveTransformation())
   ,STELLAR(new StellarTransformation())
   ,SELECT(new SelectTransformation())
+  ,RENAME(new RenameTransformation())
   ;
   FieldTransformation mapping;
   FieldTransformations(FieldTransformation mapping) {

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
new file mode 100644
index 0000000..f8b9374
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.transformation;
+
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RenameTransformation implements FieldTransformation{
+  @Override
+  public Map<String, Object> map( Map<String, Object> input
+                                , List<String> outputField
+                                , LinkedHashMap<String, Object> fieldMappingConfig
+                                , Context context
+                                , Map<String, Object>... sensorConfig
+                                )
+  {
+    if(fieldMappingConfig == null || fieldMappingConfig.isEmpty()) {
+      return input;
+    }
+    Map<String, Object> ret = new HashMap<>();
+    for(Map.Entry<String, Object> kv : input.entrySet()) {
+      Object renamed = fieldMappingConfig.get(kv.getKey());
+      if(renamed != null) {
+        //if we're renaming, then we want to copy the field to the new name
+        ret.put(renamed.toString(), kv.getValue());
+        //and remove the old field
+        ret.put(kv.getKey(), null);
+      }
+      else {
+        ret.put(kv.getKey(), kv.getValue());
+      }
+    }
+    return ret;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
index 71a0298..b7557e8 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
@@ -85,18 +85,6 @@ public class FieldTransformationTest {
    {
     "fieldTransformations" : [
           {
-           "transformation" : "IP_PROTOCOL"
-          }
-                      ]
-   }
-   */
-  @Multiline
-  public static String badConfigMissingInput;
-
-  /**
-   {
-    "fieldTransformations" : [
-          {
             "input" : "protocol"
           }
                       ]
@@ -113,10 +101,7 @@ public class FieldTransformationTest {
     Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput());
   }
 
-  @Test(expected = IllegalStateException.class)
-  public void testInValidSerde_missingInput() throws IOException {
-    SensorParserConfig.fromBytes(Bytes.toBytes(badConfigMissingInput));
-  }
+
 
   @Test(expected = IllegalStateException.class)
   public void testInValidSerde_missingMapping() throws IOException {

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
new file mode 100644
index 0000000..cacc818
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class RenameTransformationTest {
+  /**
+   {
+    "fieldTransformations" : [
+          {
+            "transformation" : "RENAME",
+            "config" : {
+              "old_field1" : "new_field1",
+              "old_field2" : "new_field2"
+                      }
+          }
+                             ]
+   }
+   */
+  @Multiline
+  public static String smoketestConfig;
+
+  @Test
+  public void smokeTest() throws Exception {
+    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(smoketestConfig));
+    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+    JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+      for(int i = 1;i <= 10;++i) {
+        put("old_field" + i, "f" + i);
+      }
+    }});
+    handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+    Assert.assertEquals("f1", input.get("new_field1"));
+    Assert.assertEquals("f2", input.get("new_field2"));
+    for(int i = 3;i <= 10;++i) {
+      Assert.assertEquals("f" + i, input.get("old_field" + i));
+    }
+    Assert.assertFalse(input.containsKey("old_field1"));
+    Assert.assertFalse(input.containsKey("old_field2"));
+    Assert.assertEquals(10, input.size());
+  }
+
+  /**
+   {
+    "fieldTransformations" : [
+          {
+            "transformation" : "RENAME",
+            "config" : {
+              "old_field1" : "new_field1"
+                      }
+          }
+                             ]
+   }
+   */
+  @Multiline
+  public static String renameMissingField;
+  @Test
+  public void renameMissingField() throws Exception {
+    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField));
+    FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+    JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+      for(int i = 2;i <= 10;++i) {
+        put("old_field" + i, "f" + i);
+      }
+    }});
+    handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+    Assert.assertFalse(input.containsKey("new_field1"));
+    for(int i = 2;i <= 10;++i) {
+      Assert.assertEquals("f" + i, input.get("old_field" + i));
+    }
+    Assert.assertEquals(9, input.size());
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index 1d2d834..e8b2896 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -313,10 +313,33 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}`
 * `STELLAR` : This transformation executes a set of transformations
   expressed as [Stellar Language](../metron-common) statements.
 
+* `RENAME` : This transformation allows users to rename a set of fields.  Specifically,
+the config is presumed to be the mapping.  The keys to the config are the existing field
names
+and the values for the config map are the associated new field name.
+
+The following config will rename the fields `old_field` and `different_old_field` to
+`new_field` and `different_new_field` respectively:
+```
+{
+...
+    "fieldTransformations" : [
+          {
+            "transformation" : "RENAME",
+          , "config" : {
+            "old_field" : "new_field",
+            "different_old_field" : "different_new_field"
+                       }
+          }
+                      ]
+}
+```
+
+
 ### Assignment to `null`
 
 If, in your field transformation, you assign a field to `null`, the field will be removed.
-You can use this capability to rename variables.
+You can use this capability to rename variables.  It is preferred, however, that the `RENAME`
+field transformation is used in this situation as it is less awkward.
 
 Consider this example:
 ```


Mime
View raw message