metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-434: JSON Parser closes apache/incubator-metron#261
Date Sat, 24 Sep 2016 05:10:24 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master c10a53eb8 -> 6834e146d


METRON-434: JSON Parser closes apache/incubator-metron#261


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/6834e146
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/6834e146
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/6834e146

Branch: refs/heads/master
Commit: 6834e146de163b5f6a441410a72fefb8db96944e
Parents: c10a53e
Author: cstella <cestella@gmail.com>
Authored: Sat Sep 24 01:09:43 2016 -0400
Committer: cstella <cestella@gmail.com>
Committed: Sat Sep 24 01:09:43 2016 -0400

----------------------------------------------------------------------
 .../data/jsonMap/parsed/jsonMapExampleParsed    |   2 +
 .../data/jsonMap/raw/jsonMapExampleOutput       |   2 +
 .../main/config/zookeeper/parsers/jsonMap.json  |   4 +
 .../metron/parsers/json/JSONMapParser.java      | 129 +++++++++++++++++++
 .../integration/JSONMapIntegrationTest.java     |  37 ++++++
 .../metron/parsers/json/JSONMapParserTest.java  | 112 ++++++++++++++++
 6 files changed, 286 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed
new file mode 100644
index 0000000..795ed45
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed
@@ -0,0 +1,2 @@
+{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ \"string\"
: \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] }","timestamp":1000000000000, "source.type":"jsonMap"
}
+{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", "source.type":"jsonMap","timestamp":1000000000000}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput
new file mode 100644
index 0000000..8064084
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput
@@ -0,0 +1,2 @@
+{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ] }
+{ "number" : 7 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
new file mode 100644
index 0000000..39e1da9
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json
@@ -0,0 +1,4 @@
+{
+  "parserClassName":"org.apache.metron.parsers.json.JSONMapParser",
+  "sensorTopic":"jsonMap"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
new file mode 100644
index 0000000..0bcf0f5
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.json;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class JSONMapParser extends BasicParser {
+  private static interface Handler {
+    JSONObject handle(String key, Map value, JSONObject obj);
+  }
+  public static enum MapStrategy implements Handler {
+     DROP((key, value, obj) -> obj)
+    ,UNFOLD( (key, value, obj) -> {
+      Set<Map.Entry<Object, Object>> entrySet = value.entrySet();
+      for(Map.Entry<Object, Object> kv : entrySet) {
+        String newKey = Joiner.on(".").join(key, kv.getKey().toString());
+        obj.put(newKey, kv.getValue());
+      }
+      return obj;
+    })
+    ,ALLOW((key, value, obj) -> {
+      obj.put(key, value);
+      return obj;
+    })
+    ,ERROR((key, value, obj) -> {
+      throw new IllegalStateException("Unable to process " + key + " => " + value + "
because value is a map.");
+    })
+    ;
+    Handler handler;
+    MapStrategy(Handler handler) {
+      this.handler = handler;
+    }
+
+    @Override
+    public JSONObject handle(String key, Map value, JSONObject obj) {
+      return handler.handle(key, value, obj);
+    }
+
+  }
+  public static final String MAP_STRATEGY_CONFIG = "mapStrategy";
+  private MapStrategy mapStrategy = MapStrategy.DROP;
+
+  @Override
+  public void configure(Map<String, Object> config) {
+    String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name());
+    mapStrategy = MapStrategy.valueOf(strategyStr);
+  }
+
+  /**
+   * Initialize the message parser.  This is done once.
+   */
+  @Override
+  public void init() {
+
+  }
+
+  /**
+   * Take raw data and convert it to a list of messages.
+   *
+   * @param rawMessage
+   * @return If null is returned, this is treated as an empty list.
+   */
+  @Override
+  public List<JSONObject> parse(byte[] rawMessage) {
+    try {
+      String originalString = new String(rawMessage);
+      //convert the JSON blob into a String -> Object map
+      Map<String, Object> rawMap = JSONUtils.INSTANCE.load(originalString, new TypeReference<Map<String,
Object>>() {
+      });
+      JSONObject ret = normalizeJSON(rawMap);
+      ret.put("original_string", originalString );
+      if(!ret.containsKey("timestamp")) {
+        //we have to ensure that we have a timestamp.  This is one of the pre-requisites
for the parser.
+        ret.put("timestamp", System.currentTimeMillis());
+      }
+      return ImmutableList.of(ret);
+    } catch (Throwable e) {
+      String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+      LOG.error(message, e);
+      throw new IllegalStateException(message, e);
+    }
+  }
+
+  /**
+   * Process all sub-maps via the MapHandler.  We have standardized on one-dimensional maps
as our data model..
+   *
+   * @param map
+   * @return
+   */
+  private JSONObject normalizeJSON(Map<String, Object> map) {
+    JSONObject ret = new JSONObject();
+    for(Map.Entry<String, Object> kv : map.entrySet()) {
+      if(kv.getValue() instanceof Map) {
+        mapStrategy.handle(kv.getKey(), (Map) kv.getValue(), ret);
+      }
+      else {
+        ret.put(kv.getKey(), kv.getValue());
+      }
+    }
+    return ret;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
new file mode 100644
index 0000000..13a7835
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.integration;
+
+import org.apache.metron.parsers.integration.validation.SampleDataValidation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class JSONMapIntegrationTest extends ParserIntegrationTest {
+  @Override
+  String getSensorType() {
+    return "jsonMap";
+  }
+
+  @Override
+  List<ParserValidation> getValidations() {
+    return new ArrayList<ParserValidation>() {{
+      add(new SampleDataValidation());
+    }};
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
new file mode 100644
index 0000000..1299d97
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.json;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class JSONMapParserTest {
+
+  /**
+   {
+     "foo" : "bar"
+    ,"blah" : "blah"
+    ,"number" : 2.0
+   }
+   */
+   @Multiline
+   static String happyPathJSON;
+
+  @Test
+  public void testHappyPath() {
+    JSONMapParser parser = new JSONMapParser();
+    List<JSONObject> output = parser.parse(happyPathJSON.getBytes());
+    Assert.assertEquals(output.size(), 1);
+    //don't forget the timestamp field!
+    Assert.assertEquals(output.get(0).size(), 5);
+    JSONObject message = output.get(0);
+    Assert.assertEquals("bar", message.get("foo"));
+    Assert.assertEquals("blah", message.get("blah"));
+    Assert.assertNotNull(message.get("timestamp"));
+    Assert.assertTrue(message.get("timestamp") instanceof Number);
+    Assert.assertNotNull(message.get("number"));
+    Assert.assertTrue(message.get("number") instanceof Number);
+  }
+
+  /**
+   {
+    "collection" : { "blah" : 7, "blah2" : "foo" }
+   }
+   */
+   @Multiline
+   static String collectionHandlingJSON;
+
+  @Test
+  public void testCollectionHandlingDrop() {
+    JSONMapParser parser = new JSONMapParser();
+    List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+    Assert.assertEquals(output.size(), 1);
+    //don't forget the timestamp field!
+    Assert.assertEquals(output.get(0).size(), 2);
+    JSONObject message = output.get(0);
+    Assert.assertNotNull(message.get("timestamp"));
+    Assert.assertTrue(message.get("timestamp") instanceof Number);
+  }
+
+  @Test(expected=IllegalStateException.class)
+  public void testCollectionHandlingError() {
+    JSONMapParser parser = new JSONMapParser();
+    parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name()));
+    parser.parse(collectionHandlingJSON.getBytes());
+  }
+
+
+  @Test
+  public void testCollectionHandlingAllow() {
+    JSONMapParser parser = new JSONMapParser();
+    parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name()));
+    List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+    Assert.assertEquals(output.size(), 1);
+    //don't forget the timestamp field!
+    Assert.assertEquals(output.get(0).size(), 3);
+    JSONObject message = output.get(0);
+    Assert.assertNotNull(message.get("timestamp"));
+    Assert.assertTrue(message.get("timestamp") instanceof Number);
+  }
+
+  @Test
+  public void testCollectionHandlingUnfold() {
+    JSONMapParser parser = new JSONMapParser();
+    parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name()));
+    List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes());
+    Assert.assertEquals(output.size(), 1);
+    //don't forget the timestamp field!
+    Assert.assertEquals(output.get(0).size(), 4);
+    JSONObject message = output.get(0);
+    Assert.assertEquals(message.get("collection.blah"), 7);
+    Assert.assertEquals(message.get("collection.blah2"), "foo");
+    Assert.assertNotNull(message.get("timestamp"));
+    Assert.assertTrue(message.get("timestamp") instanceof Number);
+  }
+}


Mime
View raw message