metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [06/50] [abbrv] metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021
Date Mon, 11 Jun 2018 21:53:30 GMT
METRON-1568: Stellar should have a _ special variable which returns the message in map form
closes apache/incubator-metron#1021


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella <cestella@gmail.com>
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Thu Jun 7 10:54:05 2018 -0400

----------------------------------------------------------------------
 .../enrichment/handler/StellarConfig.java       |  11 +-
 .../StellarTransformationTest.java              |  30 ++-
 .../adapters/stellar/StellarAdapterTest.java    |  26 ++
 .../integration/EnrichmentIntegrationTest.java  |   2 +
 .../main/config/zookeeper/enrichments/test.json |   1 +
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 metron-stellar/stellar-common/README.md         |   1 +
 .../metron/stellar/common/utils/ConcatMap.java  | 256 +++++++++++++++++++
 .../common/utils/StellarProcessorUtils.java     |  51 +++-
 .../metron/stellar/dsl/MapVariableResolver.java |   8 +
 .../metron/stellar/dsl/VariableResolver.java    |   1 +
 .../dsl/functions/DataStructureFunctions.java   |   4 +
 .../stellar/common/utils/ConcatMapTest.java     |  83 ++++++
 .../stellar/dsl/functions/BasicStellarTest.java |  18 ++
 14 files changed, 487 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package org.apache.metron.common.configuration.enrichment.handler;
 
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
   {
 
     Map<String, Object> messageSegment = new HashMap<>();
-    for(String variable : stellarFields) {
-      messageSegment.put(variable, message.get(variable));
+    if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+      //we need to include all of the fields here.
+      messageSegment.putAll(message);
+    }
+    else {
+      for (String variable : stellarFields) {
+        messageSegment.put(variable, message.get(variable));
+      }
     }
     return messageSegment;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
   @Multiline
   public static String badConfig;
 
+  /**
+   { "fieldTransformations" : [
+        { "transformation" : "STELLAR"
+        ,"output" : [ "new_field"]
+        ,"config" : {
+          "new_field" : "MAP_GET('source.type', _)"
+                    }
+        }
+                                ]
+      }
+   */
+ @Multiline
+ public static String configAll;
+
+  @Test
+  public void testConfigAll() throws Exception {
+    SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+    JSONObject input = new JSONObject();
+    input.put("source.type", "test");
+    for (FieldTransformer handler : c.getFieldTransformations()) {
+      handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+    }
+    Assert.assertEquals(2, input.size());
+    Assert.assertTrue(input.containsKey("new_field"));
+    Assert.assertEquals("test", input.get("new_field"));
+  }
 
   /** { "fieldTransformations" : [
         { "transformation" : "STELLAR"
@@ -92,8 +118,8 @@ public class StellarTransformationTest {
                                 ]
       }
    */
-  @Multiline
-  public static String configRename;
+ @Multiline
+ public static String configRename;
 
  @Test
  public void testStellarRename() throws Exception {

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
index cd1006c..e03e69f 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
@@ -165,6 +165,7 @@ public class StellarAdapterTest extends StellarEnrichmentTest {
   @Multiline
   public static String mapConfig_default;
 
+
   private void testMapEnrichment(String config, String field) throws Exception {
     JSONObject message = getMessage();
     EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class);
@@ -185,4 +186,29 @@ public class StellarAdapterTest extends StellarEnrichmentTest {
   public void testMapEnrichment_default() throws Exception {
     testMapEnrichment(mapConfig_default, "");
   }
+
+  /**
+   {
+    "fieldMap": {
+      "stellar" : {
+        "config" : [
+            "stmt1 := MAP_GET('source.type', _)"
+        ]
+      }
+    }
+  }
+   */
+  @Multiline
+  public static String allVariableConfig;
+
+  @Test
+  public void testAllVariableUsage() throws Exception {
+    JSONObject message = getMessage();
+    EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(allVariableConfig, EnrichmentConfig.class);
+    Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
+    ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
+    JSONObject enriched = enrich(message, "", handler);
+    Assert.assertEquals("stellar_test", enriched.get("stmt1"));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 2e22eab..188f18b 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -294,6 +294,8 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
     Assert.assertEquals(1, jsonDoc.get("one"));
     Assert.assertEquals(1, jsonDoc.get("map.blah"));
     Assert.assertNotNull(jsonDoc.get("foo"));
+    Assert.assertNotNull(jsonDoc.get("alt_src_type"));
+    Assert.assertEquals("test", jsonDoc.get("alt_src_type"));
     Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS"));
     Assert.assertNotNull(jsonDoc.get("bar"));
     Assert.assertEquals("TEST", jsonDoc.get("bar"));

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
index 4130943..9b997a6 100644
--- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
@@ -19,6 +19,7 @@
                       "map" : "{ 'blah' : 1}"
                       ,"one" : "MAP_GET('blah', map)"
                       ,"foo": "1 + 1"
+                      ,"alt_src_type" : "MAP_GET('source.type', _)"
                       }
           ,"ALL_CAPS" : "TO_UPPER(source.type)"
           ,"src_enrichment" : {

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java
index 4ea6836..e3ac7e5 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java
@@ -18,6 +18,8 @@
 
 package org.apache.metron.pcap.filter;
 
+import com.google.common.collect.ImmutableList;
+import org.apache.metron.stellar.common.utils.ConcatMap;
 import org.apache.metron.stellar.dsl.VariableResolver;
 
 import java.util.HashMap;
@@ -32,6 +34,9 @@ public class PcapFieldResolver implements VariableResolver {
 
   @Override
   public Object resolve(String variable) {
+    if(variable.equals(VariableResolver.ALL_FIELDS)) {
+      return new ConcatMap(ImmutableList.of(fieldsMap));
+    }
     return fieldsMap.get(variable);
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 291f1be..dcf0dbf 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -37,6 +37,7 @@ For a variety of components (threat intelligence triage and field transformation
 
 The Stellar language supports the following:
 * Referencing fields in the enriched JSON
+* Referencing all fields in the enriched JSON via the `_` reserved variable name.
 * String literals are quoted with either `'` or `"`
 * String literals support escaping for `'`, `"`, `\t`, `\r`, `\n`, and backslash 
   * The literal `'\'foo\''` would represent `'foo'`

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java
new file mode 100644
index 0000000..53a52a6
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.common.utils;
+
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.KryoSerializable;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * ConcatMap is a lazy concatenation of a list of Maps.  It is lazy in that it does not construct
+ * a union of all of the maps, but rather keeps the maps separate.  Key/Value resolution
is
+ * done via a first-wins strategy (i.e. the first map which has a key will be used).
+ *
+ * Also, note, that this is an immutable map, so operations which require mutation will have
+ * UnsupportedOperationException thrown.
+ */
+public class ConcatMap implements Map<String, Object>, Serializable, KryoSerializable
{
+  List<Map> variableMappings = new ArrayList<>();
+  public ConcatMap(List<Map> variableMappings) {
+    this.variableMappings = variableMappings;
+  }
+
+  @Override
+  public int size() {
+    int size = 0;
+    for(Map m : variableMappings) {
+      size += m.size();
+    }
+    return size;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    boolean isEmpty = true;
+    for(Map m : variableMappings) {
+      isEmpty &= m.isEmpty();
+    }
+    return isEmpty;
+  }
+
+  /**
+   * If any maps contains the key, then this will return true.
+   * @param key
+   * @return
+   */
+  @Override
+  public boolean containsKey(Object key) {
+    for(Map m : variableMappings) {
+      if(m.containsKey(key)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   *
+   * If any maps contains the value, then this will return true.
+   * @param value
+   * @return
+   */
+  @Override
+  public boolean containsValue(Object value) {
+    for(Map m : variableMappings) {
+      if(m.containsValue(value)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * The first map which contains the key will have the associated value returned.
+   * @param key
+   * @return
+   */
+  @Override
+  public Object get(Object key) {
+    Object ret = null;
+    for(Map m : variableMappings) {
+      ret = m.get(key);
+      if(ret != null) {
+        break;
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * This is an immutable map and this operation is not supported.
+   * @param key
+   * @param value
+   * @return
+   */
+  @Override
+  public Object put(String key, Object value) {
+    throw new UnsupportedOperationException("Merged map is immutable.");
+  }
+
+  /**
+   *
+   * This is an immutable map and this operation is not supported.
+   * @param key
+   * @return
+   */
+  @Override
+  public Object remove(Object key) {
+    throw new UnsupportedOperationException("Merged map is immutable.");
+  }
+
+  /**
+   *
+   * This is an immutable map and this operation is not supported.
+   * @param m
+   */
+  @Override
+  public void putAll(Map<? extends String, ?> m) {
+    throw new UnsupportedOperationException("Merged map is immutable.");
+  }
+
+  /**
+   *
+   * This is an immutable map and this operation is not supported.
+   */
+  @Override
+  public void clear() {
+    throw new UnsupportedOperationException("Merged map is immutable.");
+  }
+
+  @Override
+  public Set<String> keySet() {
+    Set<String> ret = null;
+    for(Map m : variableMappings) {
+      if(ret == null) {
+        ret = m.keySet();
+      }
+      else {
+        ret = Sets.union(ret, m.keySet());
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * Note: this makes a copy of the values, so it is not fundamentally lazy.
+   * @return
+   */
+  @Override
+  public Collection<Object> values() {
+    Collection<Object> ret = new ArrayList<>(size());
+    for(Map m : variableMappings) {
+      ret.addAll(m.values());
+    }
+    return ret;
+  }
+
+  /**
+   * This is a lazy entry collection of the associated maps.  If there are duplicate keys,
they will appear
+   * twice here, so be careful.
+   * @return
+   */
+  @Override
+  public Set<Entry<String, Object>> entrySet() {
+    Set<Entry<String, Object>> ret = null;
+    for(Map m : variableMappings) {
+      if(ret == null) {
+        ret = m.entrySet();
+      }
+      else {
+        ret = Sets.union(ret, m.entrySet());
+      }
+    }
+    return ret;
+  }
+
+  @Override
+  public String toString() {
+    Iterable<Iterable<Map.Entry<Object, Object>>> transformed =
+            Iterables.transform(variableMappings, x -> x.entrySet());
+    Iterable<Map.Entry<Object, Object>> it = Iterables.filter( Iterables.concat(transformed)
+                                                             , x -> x.getValue() != null
+                                                             );
+    return "{" + Joiner.on(", ").join(it) + "}";
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    ConcatMap concatMap = (ConcatMap) o;
+
+    return variableMappings != null ? variableMappings.equals(concatMap.variableMappings)
: concatMap.variableMappings == null;
+
+  }
+
+  @Override
+  public int hashCode() {
+    return variableMappings != null ? variableMappings.hashCode() : 0;
+  }
+
+  @Override
+  public void write(Kryo kryo, Output output) {
+    int numVariableMappings = variableMappings.isEmpty()?0:variableMappings.size();
+    output.writeShort(numVariableMappings);
+    for(Map m : variableMappings) {
+      byte[] b = m == null?new byte[]{}:SerDeUtils.toBytes(m);
+      output.writeInt(b.length);
+      if(b.length > 0) {
+        output.writeBytes(b);
+      }
+    }
+  }
+
+  @Override
+  public void read(Kryo kryo, Input input) {
+    int numVariableMappings = input.readShort();
+    variableMappings = new ArrayList<>(numVariableMappings);
+    for(int i = 0;i < numVariableMappings;++i) {
+      int size = input.readInt();
+      if(size > 0) {
+        byte[] bytes = input.readBytes(size);
+        Map m = SerDeUtils.fromBytes(bytes, Map.class);
+        variableMappings.add(m);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java
index 4ad5a40..51e7aaa 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java
@@ -62,14 +62,14 @@ public class StellarProcessorUtils {
    * validates successfully and produces a result that can be serialized correctly.
    *
    * @param expression The expression to execute.
-   * @param variables The variables to expose to the expression.
+   * @param varResolver The variable resolver to use
    * @param context The execution context.
    * @return The result of executing the expression.
    */
-  public static Object run(String expression, Map<String, Object> variables, Context
context) {
+  public static Object run(String expression, VariableResolver varResolver, Context context)
{
 
     validate(expression, context);
-    Object result = execute(expression, variables, context);
+    Object result = execute(expression, varResolver, context);
     ensureKryoSerializable(result, expression);
     ensureJavaSerializable(result, expression);
 
@@ -77,19 +77,44 @@ public class StellarProcessorUtils {
   }
 
   /**
+   * Execute and validate a Stellar expression.
+   *
+   * <p>This is intended for use while unit testing Stellar expressions.  This ensures
that the expression
+   * validates successfully and produces a result that can be serialized correctly.
+   *
+   * @param expression The expression to execute.
+   * @param variables The variables to expose to the expression.
+   * @param context The execution context.
+   * @return The result of executing the expression.
+   */
+  public static Object run(String expression, Map<String, Object> variables, Context
context) {
+    VariableResolver varResolver = new DefaultVariableResolver(
+            x -> {
+              if(x.equals(MapVariableResolver.ALL_FIELDS)) {
+                return variables;
+              }
+              return variables.get(x);
+            }
+            ,x-> x.equals(MapVariableResolver.ALL_FIELDS) || variables.containsKey(x)
+    );
+    return run(expression, varResolver, context);
+  }
+
+  /**
    * Execute a Stellar expression.
    *
    * @param expression The expression to execute.
-   * @param variables The variables available to the expression.
+   * @param variableResolver Variable Resolver to use
    * @param context The execution context.
    * @return The result of executing the expression.
    */
-  private static Object execute(String expression, Map<String, Object> variables, Context
context) {
+  private static Object execute(String expression, VariableResolver variableResolver, Context
context) {
 
     StellarProcessor processor = new StellarProcessor();
+
     Object result = processor.parse(
             expression,
-            new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)),
+            variableResolver,
             StellarFunctions.FUNCTION_RESOLVER(),
             context);
 
@@ -180,6 +205,20 @@ public class StellarProcessorUtils {
    * validates successfully and produces a result that can be serialized correctly.
    *
    * @param expression The expression to execute.
+   * @param variables The variables to expose to the expression.
+   * @return The result of executing the expression.
+   */
+  public static Object run(String expression, VariableResolver variables) {
+    return run(expression, variables, Context.EMPTY_CONTEXT());
+  }
+
+  /**
+   * Execute and validate a Stellar expression.
+   *
+   * <p>This is intended for use while unit testing Stellar expressions.  This ensures
that the expression
+   * validates successfully and produces a result that can be serialized correctly.
+   *
+   * @param expression The expression to execute.
    * @param context The execution context.
    * @return The result of executing the expression.
    */

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
index 4c02d99..872211d 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
@@ -18,12 +18,16 @@
 
 package org.apache.metron.stellar.dsl;
 
+
+import org.apache.metron.stellar.common.utils.ConcatMap;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 public class MapVariableResolver implements VariableResolver {
 
+
   List<Map> variableMappings = new ArrayList<>();
 
   public MapVariableResolver(Map variableMappingOne, Map... variableMapping) {
@@ -45,6 +49,10 @@ public class MapVariableResolver implements VariableResolver {
 
   @Override
   public Object resolve(String variable) {
+    if(variable != null && variable.equals(VariableResolver.ALL_FIELDS)) {
+      return new ConcatMap(variableMappings);
+    }
+
     for (Map variableMapping : variableMappings) {
       Object o = variableMapping.get(variable);
       if (o != null) {

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java
index f2624e9..fb95d27 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java
@@ -20,6 +20,7 @@ package org.apache.metron.stellar.dsl;
 
 
 public interface VariableResolver {
+  public static final String ALL_FIELDS = "_";
   Object resolve(String variable);
   boolean exists(String variable);
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
index eb1db48..bebe27d 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
@@ -27,6 +27,7 @@ import org.apache.metron.stellar.common.utils.SerDeUtils;
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class DataStructureFunctions {
@@ -211,6 +212,9 @@ public class DataStructureFunctions {
       if(o instanceof Collection) {
         return ((Collection)o).size();
       }
+      else if(o instanceof Map) {
+        return ((Map)o).size();
+      }
       else if(o instanceof String) {
         String val = (String) list.get(0);
         return val == null || val.isEmpty() ? 0 : val.length();

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java
new file mode 100644
index 0000000..4c078e8
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ConcatMapTest {
+
+  @Test
+  public void testToString() throws Exception {
+    Map<String, Object> v1 = new HashMap<>();
+    v1.put("k1", "v1");
+    Map<String, Object> v2 = new HashMap<>();
+    v2.put("k2", "v2");
+    v2.put("k3", null);
+    Map<String, Object> union = new HashMap<String, Object>() {{
+      putAll(v1);
+      put("k2", "v2");
+    }};
+    ConcatMap c = create(v1, v2);
+    Assert.assertEquals(c.toString(), union.toString());
+  }
+
+  private ConcatMap create(Map... ms) {
+    List<Map> l = new ArrayList<>();
+    for(Map m : ms) {
+      l.add(m);
+    }
+    return new ConcatMap(l);
+  }
+
+  private void assertKryoserializable(ConcatMap c) {
+    byte[] serialized = SerDeUtils.toBytes(c);
+    ConcatMap deserialized = SerDeUtils.fromBytes(serialized, ConcatMap.class);
+    Assert.assertEquals(deserialized, c);
+  }
+
+  @Test
+  public void testKryoSerialization() {
+    Map<String, Object> v1 = new HashMap<>();
+    v1.put("k1", "v1");
+    Map<String, Object> v2 = new HashMap<>();
+    v2.put("k2", "v2");
+    v2.put("k3", null);
+    {
+      //multi maps
+      ConcatMap c = create(v1, v2);
+      assertKryoserializable(c);
+    }
+    {
+      //single maps
+      ConcatMap c = create(v1);
+      assertKryoserializable(c);
+    }
+    {
+      //empty maps
+      ConcatMap c = create();
+      assertKryoserializable(c);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index aa4462a..ea859c5 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -25,9 +25,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.stellar.common.StellarProcessor;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.DefaultVariableResolver;
+import org.apache.metron.stellar.dsl.MapVariableResolver;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver;
 import org.junit.Assert;
 import org.junit.Ignore;
@@ -957,4 +959,20 @@ public class BasicStellarTest {
     thrown.expectMessage("The rule 'TO_UPPER(protocol)' does not return a boolean value.");
     runPredicate("TO_UPPER(protocol)", new DefaultVariableResolver(v -> variableMap.get(v),v
-> variableMap.containsKey(v)));
   }
+
+  @Test
+  public void all_fields_test() {
+    final Map<String, Object> varMap1 = new HashMap<String, Object>();
+    varMap1.put("field1", "val1");
+    final Map<String, Object> varMap2 = new HashMap<String, Object>();
+    varMap2.put("field2", "val2");
+    VariableResolver resolver = new MapVariableResolver(varMap1, varMap2);
+    Assert.assertTrue(runPredicate("MAP_GET('field1', _) == 'val1'", resolver));
+    Assert.assertTrue(runPredicate("MAP_GET('field2', _) == 'val2'", resolver));
+    Assert.assertTrue(runPredicate("LENGTH(_) == 2", resolver));
+    Map<String, Object> ret = (Map<String, Object>) run("_", resolver);
+    Assert.assertEquals(2, ret.size());
+    Assert.assertEquals("val1", ret.get("field1"));
+    Assert.assertEquals("val2", ret.get("field2"));
+  }
 }


Mime
View raw message