metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject incubator-metron git commit: METRON-372 Enhance Statistical Operations Available for Use with the Profiler (nickwallen) closes apache/incubator-metron#225
Date Wed, 24 Aug 2016 13:32:15 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 5adff7449 -> fa946169c


METRON-372 Enhance Statistical Operations Available for Use with the Profiler (nickwallen)
closes apache/incubator-metron#225


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/fa946169
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/fa946169
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/fa946169

Branch: refs/heads/master
Commit: fa946169cf0987ebddfd7e309f2bb471a6d9f18b
Parents: 5adff74
Author: nickwallen <nick@nickallen.org>
Authored: Wed Aug 24 09:31:27 2016 -0400
Committer: Nick Allen <nick@nickallen.org>
Committed: Wed Aug 24 09:31:27 2016 -0400

----------------------------------------------------------------------
 metron-analytics/metron-profiler/README.md      |  16 +-
 .../profiler/bolt/ProfileSplitterBolt.java      |   1 -
 .../zookeeper/readme-example-2/profiler.json    |   8 +-
 .../zookeeper/readme-example-3/profiler.json    |  14 +-
 .../integration/ProfilerIntegrationTest.java    |  12 +-
 metron-platform/metron-common/pom.xml           |   7 +-
 .../metron/common/dsl/StellarFunctions.java     | 120 ++++++----
 .../functions/SummaryStatisticsFunctions.java   | 231 +++++++++++++++++++
 .../StellarStatisticalFunctionsTest.java        | 184 +++++++++++++++
 metron-platform/metron-hbase/pom.xml            |  16 --
 .../org/apache/metron/test/mock/MockHTable.java |   2 +-
 11 files changed, 517 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
index f1e73f8..f1ba045 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -125,15 +125,8 @@ The average of the `length` field of HTTP traffic. The following configuration
w
       "profile": "example3",
       "foreach": "ip_src_addr",
       "onlyif": "protocol == 'HTTP'",
-      "init": {
-        "sum": 0.0,
-        "cnt": 0.0
-      },
-      "update": {
-        "sum": "sum + length",
-        "cnt": "cnt + 1"
-      },
-      "result": "sum / cnt"
+      "update": { "s": "STATS_ADD(s, length)" },
+      "result": "STATS_MEAN(s)"
     }
   ]
 }
@@ -142,9 +135,8 @@ The average of the `length` field of HTTP traffic. The following configuration
w
 This creates a profile...
  * Named ‘example3’
  * That for each IP source address
- * Only if the 'protocol' field equals 'HTTP'
- * Accumulates the sum of length
- * Accumulates the number of messages
+ * Only if the 'protocol' field is 'HTTP'
+ * Adds the `length` field from each message
  * Calculates the average as the result
 
 ### Topology Configuration

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
index 3e4c4ec..ad54f14 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -38,7 +38,6 @@ import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.reflect.generic.Types;
 
 import java.io.UnsupportedEncodingException;
 import java.util.Map;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
index 2c6a278..c17c6da 100644
--- a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
@@ -6,12 +6,12 @@
       "foreach": "ip_src_addr",
       "onlyif": "protocol == 'DNS' or protocol == 'HTTP'",
       "init": {
-        "num_dns": 1.0,
-        "num_http": 1.0
+        "num_dns": 0.0,
+        "num_http": 0.0
       },
       "update": {
-        "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)",
-        "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)"
+        "num_dns": "if protocol == 'DNS' then num_dns + 1 else num_dns",
+        "num_http": "if protocol == 'HTTP' then num_http + 1 else num_http"
       },
       "result": "num_dns / num_http"
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
index ce9d0e0..25670c5 100644
--- a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
@@ -5,16 +5,8 @@
       "profile": "example3",
       "foreach": "ip_src_addr",
       "onlyif": "protocol == 'HTTP'",
-      "init": {
-        "sum": 0.0,
-        "cnt": 0.0
-      },
-      "update": {
-        "sum": "sum + length",
-        "cnt": "cnt + 1"
-      },
-      "result": "sum / cnt"
+      "update": { "s": "STATS_ADD(s, length)" },
+      "result": "STATS_MEAN(s)"
     }
   ]
-}
-
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
index 64d5bd8..b652afc 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
@@ -202,10 +202,11 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * @param columnQual The column qualifier.
    */
   private Double readDouble(byte[] columnQual) throws IOException {
-    ResultScanner scanner = profilerTable.getScanner(Bytes.toBytes(columnFamily), columnQual);
+    final byte[] cf = Bytes.toBytes(columnFamily);
+    ResultScanner scanner = profilerTable.getScanner(cf, columnQual);
 
     for (Result result : scanner) {
-      byte[] raw = result.getValue(Bytes.toBytes(columnFamily), ProfileHBaseMapper.QVALUE);
+      byte[] raw = result.getValue(cf, ProfileHBaseMapper.QVALUE);
       return Bytes.toDouble(raw);
     }
 
@@ -217,10 +218,11 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * @param columnQual The column qualifier.
    */
   private Integer readInteger(byte[] columnQual) throws IOException {
-    ResultScanner scanner = profilerTable.getScanner(Bytes.toBytes(columnFamily), columnQual);
+    final byte[] cf = Bytes.toBytes(columnFamily);
+    ResultScanner scanner = profilerTable.getScanner(cf, columnQual);
 
     for (Result result : scanner) {
-      byte[] raw = result.getValue(Bytes.toBytes(columnFamily), ProfileHBaseMapper.QVALUE);
+      byte[] raw = result.getValue(cf, ProfileHBaseMapper.QVALUE);
       return Bytes.toInt(raw);
     }
 
@@ -246,7 +248,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
       setProperty("profiler.hbase.salt.divisor", "10");
       setProperty("profiler.hbase.table", tableName);
       setProperty("profiler.hbase.batch", "10");
-      setProperty("profiler.hbase.flush.interval.seconds", "2");
+      setProperty("profiler.hbase.flush.interval.seconds", "1");
       setProperty("hbase.provider.impl", "" + MockTableProvider.class.getName());
     }};
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index f48e7c2..139546a 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -40,7 +40,12 @@
         <dependency>
             <groupId>org.apache.metron</groupId>
             <artifactId>metron-maas-common</artifactId>
-            <version>${parent.version}</version>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-math3</artifactId>
+            <version>3.6.1</version>
         </dependency>
         <dependency>
             <groupId>commons-validator</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
index d7057e8..36c361a 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java
@@ -18,8 +18,14 @@
 
 package org.apache.metron.common.dsl;
 
-import org.apache.commons.net.util.SubnetUtils;
-import org.apache.metron.common.dsl.functions.*;
+import org.apache.metron.common.dsl.functions.ConversionFunctions;
+import org.apache.metron.common.dsl.functions.DataStructureFunctions;
+import org.apache.metron.common.dsl.functions.DateFunctions;
+import org.apache.metron.common.dsl.functions.MaaSFunctions;
+import org.apache.metron.common.dsl.functions.MapFunctions;
+import org.apache.metron.common.dsl.functions.NetworkFunctions;
+import org.apache.metron.common.dsl.functions.StringFunctions;
+import org.apache.metron.common.dsl.functions.SummaryStatisticsFunctions;
 import org.apache.metron.common.field.transformation.IPProtocolTransformation;
 import org.apache.metron.common.field.validation.network.DomainValidation;
 import org.apache.metron.common.field.validation.network.EmailValidation;
@@ -27,50 +33,78 @@ import org.apache.metron.common.field.validation.network.IPValidation;
 import org.apache.metron.common.field.validation.network.URLValidation;
 import org.apache.metron.common.field.validation.primitive.DateValidation;
 import org.apache.metron.common.field.validation.primitive.IntegerValidation;
-import org.apache.metron.common.utils.ConversionUtils;
 
 import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
 import java.util.function.Predicate;
 
 public enum StellarFunctions implements StellarFunction {
-  TO_LOWER(new StringFunctions.ToLower())
-  ,TO_UPPER(new StringFunctions.ToUpper())
-  ,TO_STRING(new StringFunctions.ToString())
-  ,TO_INTEGER(new ConversionFunctions.Cast<>(Integer.class))
-  ,TO_DOUBLE(new ConversionFunctions.Cast<>(Double.class))
-  ,TRIM(new StringFunctions.Trim())
-  ,JOIN(new StringFunctions.JoinFunction())
-  ,SPLIT(new StringFunctions.SplitFunction())
-  ,GET_FIRST(new StringFunctions.GetFirst())
-  ,GET_LAST(new StringFunctions.GetLast())
-  ,GET(new StringFunctions.Get())
-  ,MAP_GET(new MapFunctions.MapGet())
-  ,DOMAIN_TO_TLD(new NetworkFunctions.ExtractTLD())
-  ,DOMAIN_REMOVE_TLD(new NetworkFunctions.RemoveTLD())
-  ,DOMAIN_REMOVE_SUBDOMAINS(new NetworkFunctions.RemoveSubdomains())
-  ,URL_TO_HOST(new NetworkFunctions.URLToHost())
-  ,URL_TO_PORT(new NetworkFunctions.URLToPort())
-  ,URL_TO_PATH(new NetworkFunctions.URLToPath())
-  ,URL_TO_PROTOCOL(new NetworkFunctions.URLToProtocol())
-  ,TO_EPOCH_TIMESTAMP(new DateFunctions.ToTimestamp())
-  ,PROTOCOL_TO_NAME(new IPProtocolTransformation())
-  ,IS_EMPTY ( new DataStructureFunctions.IsEmpty())
-  ,IN_SUBNET( new NetworkFunctions.InSubnet())
-  ,STARTS_WITH( new StringFunctions.StartsWith())
-  ,ENDS_WITH( new StringFunctions.EndsWith())
-  ,REGEXP_MATCH( new StringFunctions.RegexpMatch())
-  , IS_IP(new Predicate2Transformation(new IPValidation()))
-  , IS_DOMAIN(new Predicate2Transformation(new DomainValidation()))
-  , IS_EMAIL(new Predicate2Transformation(new EmailValidation()))
-  , IS_URL(new Predicate2Transformation(new URLValidation()))
-  , IS_DATE(new Predicate2Transformation(new DateValidation()))
-  , IS_INTEGER(new Predicate2Transformation(new IntegerValidation()))
-  , MAP_EXISTS( new MapFunctions.MapExists())
-  , MAAS_GET_ENDPOINT( new MaaSFunctions.GetEndpoint())
-  , MODEL_APPLY(new MaaSFunctions.ModelApply())
-  ;
+
+  // string functions
+  TO_LOWER(new StringFunctions.ToLower()),
+  TO_UPPER(new StringFunctions.ToUpper()),
+  TRIM(new StringFunctions.Trim()),
+  JOIN(new StringFunctions.JoinFunction()),
+  SPLIT(new StringFunctions.SplitFunction()),
+  GET_FIRST(new StringFunctions.GetFirst()),
+  GET_LAST(new StringFunctions.GetLast()),
+  GET(new StringFunctions.Get()),
+  STARTS_WITH( new StringFunctions.StartsWith()),
+  ENDS_WITH( new StringFunctions.EndsWith()),
+  REGEXP_MATCH( new StringFunctions.RegexpMatch()),
+
+  // conversion functions
+  TO_STRING(new StringFunctions.ToString()),
+  TO_INTEGER(new ConversionFunctions.Cast<>(Integer.class)),
+  TO_DOUBLE(new ConversionFunctions.Cast<>(Double.class)),
+
+  // map functions
+  MAP_GET(new MapFunctions.MapGet()),
+  MAP_EXISTS( new MapFunctions.MapExists()),
+
+  // network functions
+  DOMAIN_TO_TLD(new NetworkFunctions.ExtractTLD()),
+  DOMAIN_REMOVE_TLD(new NetworkFunctions.RemoveTLD()),
+  DOMAIN_REMOVE_SUBDOMAINS(new NetworkFunctions.RemoveSubdomains()),
+  URL_TO_HOST(new NetworkFunctions.URLToHost()),
+  URL_TO_PORT(new NetworkFunctions.URLToPort()),
+  URL_TO_PATH(new NetworkFunctions.URLToPath()),
+  URL_TO_PROTOCOL(new NetworkFunctions.URLToProtocol()),
+  IN_SUBNET( new NetworkFunctions.InSubnet()),
+  PROTOCOL_TO_NAME(new IPProtocolTransformation()),
+
+  // date functions
+  TO_EPOCH_TIMESTAMP(new DateFunctions.ToTimestamp()),
+
+  // validation functions
+  IS_EMPTY ( new DataStructureFunctions.IsEmpty()),
+  IS_IP(new Predicate2Transformation(new IPValidation())),
+  IS_DOMAIN(new Predicate2Transformation(new DomainValidation())),
+  IS_EMAIL(new Predicate2Transformation(new EmailValidation())),
+  IS_URL(new Predicate2Transformation(new URLValidation())),
+  IS_DATE(new Predicate2Transformation(new DateValidation())),
+  IS_INTEGER(new Predicate2Transformation(new IntegerValidation())),
+
+  // model-as-a-service functions
+  MAAS_GET_ENDPOINT( new MaaSFunctions.GetEndpoint()),
+  MODEL_APPLY(new MaaSFunctions.ModelApply()),
+
+  // summary statistics
+  STATS_INIT(new SummaryStatisticsFunctions.Init()),
+  STATS_ADD(new SummaryStatisticsFunctions.Add()),
+  STATS_COUNT(new SummaryStatisticsFunctions.Count()),
+  STATS_MEAN(new SummaryStatisticsFunctions.Mean()),
+  STATS_GEOMETRIC_MEAN(new SummaryStatisticsFunctions.GeometricMean()),
+  STATS_MAX(new SummaryStatisticsFunctions.Max()),
+  STATS_MIN(new SummaryStatisticsFunctions.Min()),
+  STATS_SUM(new SummaryStatisticsFunctions.Sum()),
+  STATS_POPULATION_VARIANCE(new SummaryStatisticsFunctions.PopulationVariance()),
+  STATS_VARIANCE(new SummaryStatisticsFunctions.Variance()),
+  STATS_SECOND_MOMENT(new SummaryStatisticsFunctions.SecondMoment()),
+  STATS_QUADRATIC_MEAN(new SummaryStatisticsFunctions.QuadraticMean()),
+  STATS_SD(new SummaryStatisticsFunctions.StandardDeviation()),
+  STATS_SUM_LOGS(new SummaryStatisticsFunctions.SumLogs()),
+  STATS_SUM_SQUARES(new SummaryStatisticsFunctions.SumSquares());
+
   private static class Predicate2Transformation extends BaseStellarFunction {
     Predicate<List<Object>> pred;
     public Predicate2Transformation(Predicate<List<Object>> pred) {
@@ -82,13 +116,13 @@ public enum StellarFunctions implements StellarFunction {
       return pred.test(objects);
     }
   }
+
   StellarFunction func;
+
   StellarFunctions(StellarFunction func) {
     this.func = func;
   }
 
-
-
   @Override
   public Object apply(List<Object> input, Context context) {
     return func.apply(input, context);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
new file mode 100644
index 0000000..d8782d6
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/SummaryStatisticsFunctions.java
@@ -0,0 +1,231 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.metron.common.dsl.functions;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.metron.common.dsl.BaseStellarFunction;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.metron.common.utils.ConversionUtils.convert;
+
+/**
+ * Provides Stellar functions that can calculate summary statistics on
+ * streams of data.
+ *
+ * These functions are limited to those that can be calculated in a
+ * single-pass so that the values are not stored in memory.  This leverages
+ * the commons-math SummaryStatistics class.
+ */
+public class SummaryStatisticsFunctions {
+
+  /**
+   * Initializes the summary statistics.
+   *
+   * Initialization can occur from either STATS_INIT and STATS_ADD.
+   */
+  private static SummaryStatistics statsInit(List<Object> args) {
+    return new SummaryStatistics();
+  }
+
+  /**
+   * Initialize the summary statistics.
+   *
+   *  STATS_INIT ()
+   */
+  public static class Init extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      return statsInit(args);
+    }
+  }
+
+  /**
+   * Add an input value to those that are used to calculate the summary statistics.
+   *
+   *  STATS_ADD (stats, value [, value2, value3, ...])
+   */
+  public static class Add extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+
+      // initialize a stats object, if one does not already exist
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      if(stats == null) {
+        stats = statsInit(Collections.emptyList());
+      }
+
+      // add each of the numeric values
+      for(int i=1; i<args.size(); i++) {
+        double value = convert(args.get(i), Double.class);
+        stats.addValue(value);
+      }
+
+      return stats;
+    }
+  }
+
+  /**
+   * Calculates the mean.
+   *
+   *  STATS_MEAN (stats)
+   */
+  public static class Mean extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getMean() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the geometric mean.
+   */
+  public static class GeometricMean extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getGeometricMean() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the sum.
+   */
+  public static class Sum extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getSum() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the max.
+   */
+  public static class Max extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getMax() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the min.
+   */
+  public static class Min extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getMin() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the count of elements
+   */
+  public static class Count extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? convert(stats.getN(), Double.class) : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the population variance.
+   */
+  public static class PopulationVariance extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getPopulationVariance() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the variance.
+   */
+  public static class Variance extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getVariance() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the second moment.
+   */
+  public static class SecondMoment extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getSecondMoment() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the quadratic mean.
+   */
+  public static class QuadraticMean extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getQuadraticMean() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the standard deviation.
+   */
+  public static class StandardDeviation extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getStandardDeviation() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the sum of logs.
+   */
+  public static class SumLogs extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getSumOfLogs() : Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the sum of squares.
+   */
+  public static class SumSquares extends BaseStellarFunction {
+    @Override
+    public Object apply(List<Object> args) {
+      SummaryStatistics stats = convert(args.get(0), SummaryStatistics.class);
+      return (stats != null) ? stats.getSumsq() : Double.NaN;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
new file mode 100644
index 0000000..8019fb0
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarStatisticalFunctionsTest.java
@@ -0,0 +1,184 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.metron.common.stellar;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the statistical summary functions of Stellar.
+ */
+public class StellarStatisticalFunctionsTest {
+
+  private List<Double> values;
+  private Map<String, Object> variables;
+  private SummaryStatistics stats;
+
+  private static Object run(String expr, Map<String, Object> variables) {
+    StellarProcessor processor = new StellarProcessor();
+    assertTrue(processor.validate(expr));
+    return processor.parse(expr, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(),
Context.EMPTY_CONTEXT());
+  }
+
+  @Before
+  public void setup() {
+    variables = new HashMap<>();
+
+    // initialize the statistical summary state
+    Object result = run("STATS_INIT(0.0)", variables);
+    assertNotNull(result);
+    variables.put("stats", result);
+
+    // add some values
+    values = Arrays.asList(10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0);
+    values.stream().forEach(val -> run(format("STATS_ADD (stats, %f)", val), variables));
+
+    // add the same values to the StatisticalSummary object that is used for validation
+    stats = new SummaryStatistics();
+    values.stream().forEach(val -> stats.addValue(val));
+  }
+
+  @Test
+  public void testAddManyIntegers() throws Exception {
+    Object result = run("STATS_COUNT(stats)", variables);
+    double countAtStart = (double) result;
+
+    run("STATS_ADD(stats, 10, 20, 30, 40, 50)", variables);
+
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+  }
+
+  @Test
+  public void testAddManyFloats() throws Exception {
+    Object result = run("STATS_COUNT(stats)", variables);
+    double countAtStart = (double) result;
+
+    run("STATS_ADD(stats, 10.0, 20.0, 30.0, 40.0, 50.0)", variables);
+
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(countAtStart + 5.0, (double) actual, 0.1);
+  }
+
+  @Test
+  public void testCount() throws Exception {
+    Object actual = run("STATS_COUNT(stats)", variables);
+    assertEquals(stats.getN(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testMean() throws Exception {
+    Object actual = run("STATS_MEAN(stats)", variables);
+    assertEquals(stats.getMean(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testGeometricMean() throws Exception {
+    Object actual = run("STATS_GEOMETRIC_MEAN(stats)", variables);
+    assertEquals(stats.getGeometricMean(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testMax() throws Exception {
+    Object actual = run("STATS_MAX(stats)", variables);
+    assertEquals(stats.getMax(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testMin() throws Exception {
+    Object actual = run("STATS_MIN(stats)", variables);
+    assertEquals(stats.getMin(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSum() throws Exception {
+    Object actual = run("STATS_SUM(stats)", variables);
+    assertEquals(stats.getSum(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testPopulationVariance() throws Exception {
+    Object actual = run("STATS_POPULATION_VARIANCE(stats)", variables);
+    assertEquals(stats.getPopulationVariance(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testQuadraticMean() throws Exception {
+    Object actual = run("STATS_QUADRATIC_MEAN(stats)", variables);
+    assertEquals(stats.getQuadraticMean(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSecondMoment() throws Exception {
+    Object actual = run("STATS_SECOND_MOMENT(stats)", variables);
+    assertEquals(stats.getSecondMoment(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSumLogs() throws Exception {
+    Object actual = run("STATS_SUM_LOGS(stats)", variables);
+    assertEquals(stats.getSumOfLogs(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testStandardDeviation() throws Exception {
+    Object actual = run("STATS_SD(stats)", variables);
+    assertEquals(stats.getStandardDeviation(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testSumSquares() throws Exception {
+    Object actual = run("STATS_SUM_SQUARES(stats)", variables);
+    assertEquals(stats.getSumsq(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testVariance() throws Exception {
+    Object actual = run("STATS_VARIANCE(stats)", variables);
+    assertEquals(stats.getVariance(), (Double) actual, 0.1);
+  }
+
+  @Test
+  public void testWithNull() throws Exception {
+    Object actual = run("STATS_MEAN(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+
+    actual = run("STATS_COUNT(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+
+    actual = run("STATS_VARIANCE(null)", variables);
+    assertTrue(((Double)actual).isNaN());
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml
index 31f2960..330b52c 100644
--- a/metron-platform/metron-hbase/pom.xml
+++ b/metron-platform/metron-hbase/pom.xml
@@ -146,22 +146,6 @@
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>org.apache.storm</groupId>
-            <artifactId>storm-hbase</artifactId>
-            <version>${global_storm_version}</version>
-            <scope>provided</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.storm</groupId>
-                    <artifactId>storm-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-client</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-server</artifactId>
             <version>${global_hbase_version}</version>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa946169/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/mock/MockHTable.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/mock/MockHTable.java
b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/mock/MockHTable.java
index 75d8104..c522f8d 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/mock/MockHTable.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/mock/MockHTable.java
@@ -200,7 +200,7 @@ public class MockHTable implements HTableInterface {
   @Override
   public void batch(List<? extends Row> list, Object[] objects) throws IOException,
InterruptedException {
     Object[] results = batch(list);
-    System.arraycopy(results, 0, objects, 0, objects.length);
+    System.arraycopy(results, 0, objects, 0, results.length);
   }
 
   /**


Mime
View raw message