metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [1/2] incubator-metron git commit: METRON-189: Add the ability to do global validations on messages passing through the parser. This closes apache/incubator-metron#138
Date Thu, 02 Jun 2016 00:10:34 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master d3efe3fb4 -> 025f64c4f


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
new file mode 100644
index 0000000..ebbca66
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.validation.primitive;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.field.validation.BaseValidationTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.metron.common.query.QueryParserTest.run;
+
+public class IntegerValidationTest extends BaseValidationTest{
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : "field1"
+             ,"validation" : "INTEGER"
+            }
+                         ]
+   }
+   */
+  @Multiline
+  public static String validWithSingleField;
+  public static String validWithSingleField_MQL = "IS_INTEGER(field1)";
+
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : [ "field1", "field2" ]
+             ,"validation" : "INTEGER"
+            }
+                         ]
+   }
+   */
+  @Multiline
+  public static String validWithMultipleFields;
+  public static String validWithMultipleFields_MQL = "IS_INTEGER(field1) and IS_INTEGER(field2)";
+
+  @Test
+  public void positiveTest_single() throws IOException {
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", 1)));
+    Assert.assertTrue(run(validWithSingleField_MQL, ImmutableMap.of("field1", 1)));
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", "1")));
+    Assert.assertTrue(run(validWithSingleField_MQL, ImmutableMap.of("field1", "1")));
+  }
+  @Test
+  public void negativeTest_single() throws IOException {
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", "foo")));
+    Assert.assertFalse(run(validWithSingleField_MQL, ImmutableMap.of("field1", "foo")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", 2.3f)));
+    Assert.assertFalse(run(validWithSingleField_MQL, ImmutableMap.of("field1", 2.3f)));
+  }
+  @Test
+  public void positiveTest_multiple() throws IOException {
+    Assert.assertTrue(execute(validWithMultipleFields, ImmutableMap.of("field1", 1, "field2",
"2")));
+    Assert.assertTrue(run(validWithMultipleFields_MQL, ImmutableMap.of("field1", 1, "field2",
"2")));
+  }
+
+  @Test
+  public void negativeTest_multiple() throws IOException {
+
+    Assert.assertFalse(execute(validWithMultipleFields, ImmutableMap.of("field2", "foo")));
+    Assert.assertFalse(run(validWithMultipleFields_MQL, ImmutableMap.of("field2", "foo")));
+    Assert.assertFalse(execute(validWithMultipleFields, ImmutableMap.of("field1", "", "field2",
1)));
+    Assert.assertFalse(run(validWithMultipleFields_MQL, ImmutableMap.of("field1", "", "field2",
1)));
+    Assert.assertFalse(execute(validWithMultipleFields, ImmutableMap.of("field1", " ", "field2",
2)));
+    Assert.assertFalse(run(validWithMultipleFields_MQL, ImmutableMap.of("field1", " ", "field2",
2)));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidationTest.java
new file mode 100644
index 0000000..cbb469e
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/NotEmptyValidationTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.validation.primitive;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.FieldValidator;
+import org.apache.metron.common.field.validation.BaseValidationTest;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+
+public class NotEmptyValidationTest extends BaseValidationTest {
+
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : "field1"
+             ,"validation" : "NOT_EMPTY"
+            }
+                         ]
+   }
+   */
+  @Multiline
+  public static String validWithSingleField;
+
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : [ "field1", "field2" ]
+             ,"validation" : "NOT_EMPTY"
+            }
+                         ]
+   }
+   */
+  @Multiline
+  public static String validWithMultipleFields;
+
+  @Test
+  public void positiveTest_single() throws IOException {
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", "foo")));
+  }
+  @Test
+  public void negativeTest_single() throws IOException {
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field2", "foo")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", "")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", " ")));
+  }
+  @Test
+  public void positiveTest_multiple() throws IOException {
+    Assert.assertTrue(execute(validWithMultipleFields, ImmutableMap.of("field1", "foo", "field2",
"bar")));
+  }
+
+  @Test
+  public void negativeTest_multiple() throws IOException {
+
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field2", "foo")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", "", "field2",
"bar")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", " ", "field2",
"bar")));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/RegexValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/RegexValidationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/RegexValidationTest.java
new file mode 100644
index 0000000..8b602e9
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/RegexValidationTest.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.validation.primitive;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.field.validation.BaseValidationTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class RegexValidationTest extends BaseValidationTest {
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : "field1"
+             ,"validation" : "REGEX_MATCH"
+             ,"config" : {
+                  "pattern" : "fo.*"
+                         }
+            }
+                         ]
+   }
+   */
+
+  @Multiline
+  public static String validWithSingleField;
+
+  /**
+   {
+    "fieldValidations" : [
+            {
+              "input" : [ "field1", "field2" ]
+             ,"validation" : "REGEX_MATCH"
+             ,"config" : {
+                  "pattern" : "fo.*"
+                         }
+            }
+                         ]
+   }
+   */
+  @Multiline
+  public static String validWithMultipleFields;
+
+  @Test
+  public void positiveTest_single() throws IOException {
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", "foo")));
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", "fop")));
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field1", "fo")));
+  }
+  @Test
+  public void negativeTest_single() throws IOException {
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", "flo")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", 2.3f)));
+  }
+  @Test
+  public void positiveTest_multiple() throws IOException {
+    Assert.assertTrue(execute(validWithMultipleFields, ImmutableMap.of("field1", "fooo",
"field2", "foll")));
+  }
+
+  @Test
+  public void negativeTest_multiple() throws IOException {
+
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field2", "foo")));
+    Assert.assertFalse(execute(validWithSingleField, ImmutableMap.of("field1", 1, "field2",
"foo")));
+    Assert.assertTrue(execute(validWithSingleField, ImmutableMap.of("field3", "foo")));
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-common/src/test/java/org/apache/metron/common/query/QueryParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/query/QueryParserTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/query/QueryParserTest.java
index 4660002..1e5fbbf 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/query/QueryParserTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/query/QueryParserTest.java
@@ -45,7 +45,10 @@ public class QueryParserTest {
     }
   }
 
-  private static boolean run(String rule, VariableResolver resolver) {
+  public static boolean run(String rule, Map resolver) {
+    return run(rule, new MapVariableResolver(resolver));
+  }
+  public static boolean run(String rule, VariableResolver resolver) {
     PredicateProcessor processor = new PredicateProcessor();
     Assert.assertTrue(rule + " not valid.", processor.validate(rule));
     return processor.parse(rule, resolver);
@@ -158,4 +161,5 @@ public class QueryParserTest {
     Assert.assertTrue(run("not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", v-> variableMap.get(v)));
   }
 
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
index 721f70f..d294da7 100644
--- a/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
@@ -6,5 +6,11 @@
   "solr.zookeeper": "localhost:2181",
   "solr.collection": "metron",
   "solr.numShards": 1,
-  "solr.replicationFactor": 1
+  "solr.replicationFactor": 1,
+  "fieldValidations" : [
+    {
+       "input" : [ "src_ip_addr", "dst_ip_addr"]
+      ,"validation" : "IP"
+    }
+                       ]
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
index 442e47c..39eca58 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/FluxTopologyComponent.java
@@ -97,7 +97,9 @@ public class FluxTopologyComponent implements InMemoryComponent {
     }
 
     public void stop() {
-        stormCluster.shutdown();
+        if(stormCluster != null) {
+            stormCluster.shutdown();
+        }
     }
 
     public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException,
TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/025f64c4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index b68bac5..e3b5fd0 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -22,8 +22,10 @@ import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredParserBolt;
+import org.apache.metron.common.configuration.FieldValidator;
 import org.apache.metron.common.configuration.ParserConfigurations;
 import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
 import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
@@ -140,10 +142,15 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable
{
     try {
       boolean ackTuple = true;
       if(sensorParserConfig != null) {
+        List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
         List<JSONObject> messages = parser.parse(originalMessage);
         for (JSONObject message : messages) {
           if (parser.validate(message)) {
-            if (filter != null && filter.emitTuple(message)) {
+            if(!isGloballyValid(message, fieldValidations)) {
+              message.put(Constants.SENSOR_TYPE, getSensorType()+ ".invalid");
+              collector.emit(Constants.INVALID_STREAM, new Values(message));
+            }
+            else if (filter != null && filter.emitTuple(message)) {
               ackTuple = !isBulk;
               message.put(Constants.SENSOR_TYPE, getSensorType());
               for (FieldTransformer handler : sensorParserConfig.getFieldTransformations())
{
@@ -165,8 +172,18 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable
{
     }
   }
 
+  private boolean isGloballyValid(JSONObject input, List<FieldValidator> validators)
{
+    for(FieldValidator validator : validators) {
+      if(!validator.isValid(input, getConfigurations().getGlobalConfig())) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declareStream(Constants.INVALID_STREAM, new Fields("message"));
     declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
   }
 }


Mime
View raw message