metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-1146: Add ability to parse JSON string into JSONObject for stellar closes apache/incubator-metron#727
Date Tue, 12 Sep 2017 16:27:44 GMT
Repository: metron
Updated Branches:
  refs/heads/master 95db8b40e -> 309d3757d


METRON-1146: Add ability to parse JSON string into JSONObject for stellar closes apache/incubator-metron#727


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/309d3757
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/309d3757
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/309d3757

Branch: refs/heads/master
Commit: 309d3757df50a2dda0ac4396750e292e76926c5d
Parents: 95db8b4
Author: Anand Subramanian <anandsubbu@gmail.com>
Authored: Tue Sep 12 09:27:09 2017 -0700
Committer: cstella <cestella@gmail.com>
Committed: Tue Sep 12 09:27:09 2017 -0700

----------------------------------------------------------------------
 metron-stellar/stellar-common/README.md         |  26 +-
 .../stellar/dsl/functions/StringFunctions.java  | 115 ++++++++
 .../dsl/functions/StringFunctionsTest.java      | 261 +++++++++++++++++++
 3 files changed, 401 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 8552138..d464d8c 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -224,6 +224,9 @@ In the core language functions, we support basic functional programming
primitiv
 | [ `TO_EPOCH_TIMESTAMP`](#to_epoch_timestamp)                                          
            |
 | [ `TO_FLOAT`](#to_float)                                                              
            |
 | [ `TO_INTEGER`](#to_integer)                                                          
            |
+| [ `TO_JSON_LIST`](#to_json_List)                                                      
            |
+| [ `TO_JSON_MAP`](#to_json_map)                                                        
            |
+| [ `TO_JSON_OBJECT`](#to_json_object)                                                  
            |
 | [ `TO_LONG`](#to_long)                                                                
            |
 | [ `TO_LOWER`](#to_lower)                                                              
            |
 | [ `TO_STRING`](#to_string)                                                            
            |
@@ -638,6 +641,27 @@ In the core language functions, we support basic functional programming
primitiv
   * Description: Returns a list of the encodings that are currently supported.
   * Returns: A List of String
  
+### `TO_JSON_LIST`
+  * Description: Accepts JSON string as an input and returns a List object parsed by Jackson.
You need to be aware of content of JSON string that is to be parsed.
+  For e.g. `GET_FIRST( TO_JSON_LIST(  '[ "foo", 2]')` would yield `foo`
+  * Input:
+    * string - The JSON string to be parsed
+  * Returns: A parsed List object
+
+### `TO_JSON_MAP`
+  * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson.
You need to be aware of content of JSON string that is to be parsed.
+  For e.g. `MAP_GET( 'bar', TO_JSON_MAP(  '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+  * Input:
+    * string - The JSON string to be parsed
+  * Returns: A parsed Map object
+
+### `TO_JSON_OBJECT`
+  * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson.
You need to be aware of content of JSON string that is to be parsed.
+  For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT(  '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+  * Input:
+    * string - The JSON string to be parsed
+  * Returns: A parsed JSON object
+
 ### `LOG2`
   * Description: Returns the log (base `2`) of a number.
   * Input:
@@ -698,7 +722,7 @@ In the core language functions, we support basic functional programming
primitiv
   * Input:
     * dateTime - The datetime as a long representing the milliseconds since unix epoch
   * Returns: The current month (0-based).
-
+  
 ### `MULTISET_ADD`
   * Description: Adds to a multiset, which is a map associating objects to their instance
counts.
   * Input:

http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
index 7f4f396..4dc4790 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
@@ -18,15 +18,20 @@
 
 package org.apache.metron.stellar.dsl.functions;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.stellar.common.utils.JSONUtils;
 import org.apache.metron.stellar.dsl.BaseStellarFunction;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.common.utils.ConversionUtils;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -502,4 +507,114 @@ public class StringFunctions {
     }
   }
 
+  @Stellar(name = "TO_JSON_OBJECT"
+          , description = "Returns a JSON object for the specified JSON string"
+          , params = {
+            "str - the JSON String to convert, may be null"
+          }
+          , returns = "an Object containing the parsed JSON string"
+  )
+  public static class ToJsonObject extends BaseStellarFunction {
+
+    @Override
+    public Object apply(List<Object> strings) {
+
+      if (strings == null || strings.size() == 0) {
+        throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage:
TO_JSON_OBJECT <String>");
+      }
+      String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+      if (var == null) {
+        return null;
+      } else if (var.length() == 0) {
+        return var;
+      } else {
+        if (!(strings.get(0) instanceof String)) {
+          throw new ParseException("Valid JSON string not supplied");
+        }
+        // Return JSON Object
+        try {
+          return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class);
+        } catch (JsonProcessingException ex) {
+          throw new ParseException("Valid JSON string not supplied", ex);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+      return new ParseException("Unable to parse JSON string");
+    }
+  }
+
+  @Stellar(name = "TO_JSON_MAP"
+          , description = "Returns a MAP object for the specified JSON string"
+          , params = {
+          "str - the JSON String to convert, may be null"
+  }
+          , returns = "a MAP object containing the parsed JSON string"
+  )
+  public static class ToJsonMap extends BaseStellarFunction {
+
+    @Override
+    public Object apply(List<Object> strings) {
+
+      if (strings == null || strings.size() == 0) {
+        throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP
<JSON String>");
+      }
+      String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+      if (var == null) {
+        return null;
+      } else if (var.length() == 0) {
+        return var;
+      } else {
+        if (!(strings.get(0) instanceof String)) {
+          throw new ParseException("Valid JSON string not supplied");
+        }
+        // Return parsed JSON Object as a HashMap
+        try {
+          return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String,
Object>>(){});
+        } catch (JsonProcessingException ex) {
+          throw new ParseException("Valid JSON string not supplied", ex);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+      return new ParseException("Unable to parse JSON string");
+    }
+  }
+
+  @Stellar(name = "TO_JSON_LIST"
+          , description = "Returns a List object for the specified JSON string"
+          , params = {
+          "str - the JSON String to convert, may be null"
+  }
+          , returns = "a List object containing the parsed JSON string"
+  )
+  public static class ToJsonList extends BaseStellarFunction {
+
+    @Override
+    public Object apply(List<Object> strings) {
+
+      if (strings == null || strings.size() == 0) {
+        throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST
<JSON String>");
+      }
+      String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+      if (var == null) {
+        return null;
+      } else if (var.length() == 0) {
+        return var;
+      } else {
+        if (!(strings.get(0) instanceof String)) {
+          throw new ParseException("Valid JSON string not supplied");
+        }
+        // Return parsed JSON Object as a List
+        try {
+          return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){});
+        } catch (JsonProcessingException ex) {
+          throw new ParseException("Valid JSON string not supplied", ex);
+        } catch (IOException e) {
+          e.printStackTrace();
+          throw new ParseException("Valid JSON string not supplied", e);
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
index cb33eaa..418bf2d 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
@@ -20,15 +20,18 @@ package org.apache.metron.stellar.dsl.functions;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.collections4.map.HashedMap;
 import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
@@ -484,4 +487,262 @@ public class StringFunctionsTest {
     Assert.assertTrue(thrown);
 
   }
+
+  /**
+   * TO_JSON_OBJECT StringFunction
+   */
+
+  // Input strings to be used
+  /**
+   { "foo" : 2 }
+   */
+  @Multiline
+  private String string1;
+
+  /**
+   {
+     "foo" : "abc",
+     "bar" : "def"
+   }
+   */
+  @Multiline
+  private String string2;
+
+  /**
+   [ "foo", 2 ]
+   */
+  @Multiline
+  private String string3;
+
+  /**
+   [ "foo", "bar", "car" ]
+   */
+  @Multiline
+  private String string4;
+
+  /**
+   [
+     {
+       "foo1":"abc",
+       "bar1":"def"
+     },
+     {
+       "foo2":"ghi",
+       "bar2":"jkl"
+     }
+   ]
+   */
+  @Multiline
+  private String string5;
+
+  @Test
+  public void testToJsonObject() throws Exception {
+    //JSON Object
+    Object ret1 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string1));
+    Assert.assertNotNull(ret1);
+    Assert.assertTrue (ret1 instanceof HashMap);
+
+    Object ret2 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string2));
+    Assert.assertNotNull(ret2);
+    Assert.assertTrue (ret2 instanceof HashMap);
+    Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval",
ret2)));
+
+    //Simple Arrays
+    Object ret3 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string3));
+    Assert.assertNotNull(ret3);
+    Assert.assertTrue (ret3 instanceof ArrayList);
+    List<Object> result3 = (List<Object>) ret3;
+    Assert.assertEquals(2, result3.get(1));
+
+    Object ret4 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string4));
+    Assert.assertNotNull(ret4);
+    Assert.assertTrue (ret4 instanceof ArrayList);
+    List<Object> result4 = (List<Object>) ret4;
+    Assert.assertEquals("car", result4.get(2));
+
+    //JSON Array
+    Object ret5 = run( "TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string5));
+    Assert.assertNotNull(ret5);
+    Assert.assertTrue (ret5 instanceof ArrayList);
+    List<List<Object>> result5 = (List<List<Object>>) ret5;
+    HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+    Assert.assertEquals("def", results5Map1.get("bar1"));
+    HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+    Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+    // No input
+    boolean thrown = false;
+    try {
+      run("TO_JSON_OBJECT()", Collections.emptyMap());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+
+    // Invalid input
+    try {
+      run("TO_JSON_OBJECT('123, 456')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+
+    // Malformed JSON String
+    try {
+      run("TO_JSON_OBJECT('{\"foo\" : 2')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+  }
+
+  @Test
+  public void testToJsonMap() throws Exception {
+    //JSON Object
+    Object ret1 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string1));
+    Assert.assertNotNull(ret1);
+    Assert.assertTrue (ret1 instanceof HashMap);
+
+    Object ret2 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string2));
+    Assert.assertNotNull(ret2);
+    Assert.assertTrue (ret2 instanceof HashMap);
+    Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval",
ret2)));
+
+    //Simple Arrays
+    boolean thrown = false;
+    try {
+      run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3));
+    } catch (ParseException pe) {
+      thrown = true;
+    }
+    Assert.assertTrue(thrown);
+
+    thrown = false;
+    try {
+      run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string4));
+    } catch (ParseException pe) {
+      thrown = true;
+    }
+    Assert.assertTrue (thrown);
+
+    //JSON Array
+    thrown = false;
+    try {
+      run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string5));
+    } catch (ParseException pe) {
+      thrown = true;
+    }
+    Assert.assertTrue(thrown);
+
+
+    // No input
+    try {
+      run("TO_JSON_MAP()", Collections.emptyMap());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+
+    // Invalid input
+    try {
+      run("TO_JSON_MAP('123, 456')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+
+    // Malformed JSON String
+    try {
+      run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+    thrown = false;
+  }
+
+  @Test
+  public void testToJsonList() throws Exception {
+    //Simple Arrays
+    Object ret3 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string3));
+    Assert.assertNotNull(ret3);
+    Assert.assertTrue (ret3 instanceof ArrayList);
+    List<Object> result3 = (List<Object>) ret3;
+    Assert.assertEquals(2, result3.get(1));
+
+    Object ret4 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string4));
+    Assert.assertNotNull(ret4);
+    Assert.assertTrue (ret4 instanceof ArrayList);
+    List<Object> result4 = (List<Object>) ret4;
+    Assert.assertEquals("car", result4.get(2));
+
+    //JSON Array
+    Object ret5 = run( "TO_JSON_LIST(msg)", ImmutableMap.of("msg", string5));
+    Assert.assertNotNull(ret5);
+    Assert.assertTrue (ret5 instanceof ArrayList);
+    List<List<Object>> result5 = (List<List<Object>>) ret5;
+    HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+    Assert.assertEquals("def", results5Map1.get("bar1"));
+    HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+    Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+    //JSON Object - throws exception
+    boolean thrown = false;
+    try {
+      run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string1));
+    } catch (ParseException pe) {
+      thrown = true;
+    }
+    Assert.assertTrue(thrown);
+
+    thrown = false;
+    try {
+      run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string2));
+    } catch (ParseException pe) {
+      thrown = true;
+    }
+    Assert.assertTrue (thrown);
+
+    // No input
+    thrown = false;
+    try {
+      run("TO_JSON_LIST()", Collections.emptyMap());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+    }
+    Assert.assertTrue(thrown);
+
+    // Invalid input
+    thrown = false;
+    try {
+      run("TO_JSON_LIST('123, 456')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+
+    // Malformed JSON String
+    thrown = false;
+    try {
+      run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>());
+    } catch (ParseException pe) {
+      thrown = true;
+      Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+    }
+    Assert.assertTrue(thrown);
+  }
+
 }


Mime
View raw message