metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-712: Separate evaluation from parsing in Stellar this closes apache/incubator-metron#473
Date Thu, 09 Mar 2017 18:46:51 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 22591a7cb -> 29a0c1867


METRON-712: Separate evaluation from parsing in Stellar this closes apache/incubator-metron#473


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/29a0c186
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/29a0c186
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/29a0c186

Branch: refs/heads/master
Commit: 29a0c186788bcbd5d1147c91bf03d654e1f04245
Parents: 22591a7
Author: cstella <cestella@gmail.com>
Authored: Thu Mar 9 13:46:39 2017 -0500
Committer: cstella <cestella@gmail.com>
Committed: Thu Mar 9 13:46:39 2017 -0500

----------------------------------------------------------------------
 .../metron/profiler/client/GetProfileTest.java  |   2 +-
 .../StellarStatisticsFunctionsTest.java         |   4 +-
 metron-platform/metron-common/README.md         |  57 +++
 .../common/stellar/BaseStellarProcessor.java    |  77 ++--
 .../metron/common/stellar/StellarCompiler.java  | 369 +++++++++++--------
 .../stellar/StellarPredicateProcessor.java      |   5 +
 .../metron/common/stellar/StellarProcessor.java |   6 +
 .../stellar/benchmark/Microbenchmark.java       |  68 ++++
 .../benchmark/StellarMicrobenchmark.java        | 271 ++++++++++++++
 .../common/stellar/StellarCompilerTest.java     | 157 --------
 .../common/stellar/StellarInterpreterTest.java  | 172 +++++++++
 .../metron/common/stellar/StellarTest.java      |   3 +
 .../management/IndexingConfigFunctionsTest.java |   6 +-
 .../management/ThreatTriageFunctionsTest.java   |   2 +-
 14 files changed, 858 insertions(+), 341 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
index e1ebdbd..4bc3167 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
@@ -282,7 +282,7 @@ public class GetProfileTest {
   /**
    * Initialization should fail if the required context values are missing.
    */
-  @Test(expected = ParseException.class)
+  @Test(expected = IllegalStateException.class)
   public void testMissingContext() {
     Context empty = Context.EMPTY_CONTEXT();
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
index 5eee84b..029edb9 100644
--- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
+++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java
@@ -172,7 +172,7 @@ public class StellarStatisticsFunctionsTest {
     values.stream().forEach(val -> run(format("STATS_ADD (stats, %f)", val), variables));
   }
 
-  @Test(expected=ParseException.class)
+  @Test(expected=IllegalStateException.class)
   public void testOverflow() throws Exception {
    run(format("STATS_ADD(STATS_INIT(), %f)", (Double.MAX_VALUE + 1)), new HashMap<>());
   }
@@ -330,7 +330,7 @@ public class StellarStatisticsFunctionsTest {
     assertEquals(summaryStats.getSumOfLogs(), (Double) actual, 0.1);
   }
 
-  @Test(expected = ParseException.class)
+  @Test(expected = UnsupportedOperationException.class)
   public void testSumLogsWithWindow() throws Exception {
     statsInit(100);
     run("STATS_SUM_LOGS(stats)", variables);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index d66ab0c..c17bb5b 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -5,6 +5,7 @@
 * [Stellar Language](#stellar-language)
     * [Stellar Language Keywords](#stellar-language-keywords)
     * [Stellar Core Functions](#stellar-core-functions)
+    * [Stellar Benchmarks](#stellar-benchmarks)
     * [Stellar Shell](#stellar-shell)
 * [Global Configuration](#global-configuration)
 * [Management Utility](#management-utility)
@@ -626,6 +627,62 @@ This will convert the timestamp field to an epoch timestamp based on the
 * The value in `dc2tz` associated with the value associated with field
   `dc`, defaulting to `UTC`
 
+## Stellar Benchmarks
+
+A microbenchmarking utility is included to assist in executing microbenchmarks for Stellar functions.
+The utility can be executed via maven using the `exec` plugin, like so, from the `metron-common` directory:
+
+```
+mvn -DskipTests clean package && \
+mvn exec:java -Dexec.mainClass="org.apache.metron.common.stellar.benchmark.StellarMicrobenchmark" -Dexec.args="..."
+ ```
+where `exec.args` can be one of the following:
+```
+    -e,--expressions <FILE>   Stellar expressions
+    -h,--help                 Generate Help screen
+    -n,--num_times <NUM>      Number of times to run per expression (after
+                              warmup). Default: 1000
+    -o,--output <FILE>        File to write output.
+    -p,--percentiles <NUM>    Percentiles to calculate per run. Default:
+                              50.0,75.0,95.0,99.0
+    -v,--variables <FILE>     File containing a JSON Map of variables to use
+    -w,--warmup <NUM>         Number of times for warmup per expression.
+                              Default: 100
+```
+
+For instance, to run with a set of Stellar expression in file `/tmp/expressions.txt`:
+```
+ # simple functions
+ TO_UPPER('casey')
+ TO_LOWER(name)
+ # math functions
+ 1 + 2*(3 + int_num) / 10.0
+ 1.5 + 2*(3 + double_num) / 10.0
+ # conditionals
+ if ('foo' in ['foo']) OR one == very_nearly_one then 'one' else 'two'
+ 1 + 2*(3 + int_num) / 10.0
+ #Network funcs
+ DOMAIN_TO_TLD(domain)
+ DOMAIN_REMOVE_SUBDOMAINS(domain)
+```
+And variables in file `/tmp/variables.json`:
+```
+{
+  "name" : "casey",
+  "int_num" : 1,
+  "double_num" : 17.5,
+  "one" : 1,
+  "very_nearly_one" : 1.000001,
+  "domain" : "www.google.com"
+}
+```
+
+Written to file `/tmp/output.txt` would be the following command:
+```
+mvn -DskipTests clean package && \
+mvn exec:java -Dexec.mainClass="org.apache.metron.common.stellar.benchmark.StellarMicrobenchmark" \
+-Dexec.args="-e /tmp/expressions.txt -v /tmp/variables.json -o ./output.json"
+ ```
 ## Stellar Shell
 
 A REPL (Read Eval Print Loop) for the Stellar language that helps in debugging, troubleshooting and learning Stellar.  The Stellar DSL (domain specific language) is used to act upon streaming data within Apache Storm.  It is difficult to troubleshoot Stellar when it can only be executed within a Storm topology.  This REPL is intended to help mitigate that problem by allowing a user to replicate data encountered in production, isolate initialization errors, or understand function resolution problems.

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java
index 5d206b3..afd92b8 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java
@@ -18,6 +18,10 @@
 
 package org.apache.metron.common.stellar;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.TokenStream;
@@ -25,6 +29,8 @@ import org.antlr.v4.runtime.TokenStream;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.Stack;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.metron.common.dsl.Context;
 import org.apache.metron.common.dsl.ErrorListener;
@@ -57,8 +63,28 @@ public class BaseStellarProcessor<T> {
   /**
    * @param clazz The class containing the type that the Stellar expression being processed will evaluate to.
    */
+  Cache<String, StellarCompiler.Expression> expressionCache;
+
+  public static final int DEFAULT_CACHE_SIZE = 500;
+  public static final int DEFAULT_EXPIRY_TIME = 10;
+  public static final TimeUnit DEFAULT_EXPIRY_TIME_UNITS = TimeUnit.MINUTES;
+
   BaseStellarProcessor(final Class<T> clazz) {
+    this(clazz, DEFAULT_CACHE_SIZE, DEFAULT_EXPIRY_TIME, DEFAULT_EXPIRY_TIME_UNITS);
+  }
+
+  BaseStellarProcessor(final Class<T> clazz, int cacheSize, int expiryTime, TimeUnit expiryUnit) {
     this.clazz = clazz;
+    CacheLoader<String, StellarCompiler.Expression> loader = new CacheLoader<String, StellarCompiler.Expression>() {
+      @Override
+      public StellarCompiler.Expression load(String key) throws Exception {
+        return compile(key);
+      }
+    };
+    expressionCache = CacheBuilder.newBuilder()
+                              .maximumSize(cacheSize)
+                              .expireAfterAccess(expiryTime, expiryUnit)
+                              .build(loader);
   }
 
   /**
@@ -71,28 +97,13 @@ public class BaseStellarProcessor<T> {
     if (rule == null || isEmpty(rule.trim())) {
       return null;
     }
-    ANTLRInputStream input = new ANTLRInputStream(rule);
-    StellarLexer lexer = new StellarLexer(input);
-    lexer.removeErrorListeners();
-    lexer.addErrorListener(new ErrorListener());
-    TokenStream tokens = new CommonTokenStream(lexer);
-    StellarParser parser = new StellarParser(tokens);
-    final Set<String> ret = new HashSet<>();
-    parser.addParseListener(new StellarBaseListener() {
-      @Override
-      public void exitVariable(final StellarParser.VariableContext ctx) {
-        ret.add(ctx.getText());
-      }
-      @Override
-      public void exitExistsFunc(final StellarParser.ExistsFuncContext ctx) {
-        String variable = ctx.getChild(2).getText();
-        ret.add(variable);
-      }
-    });
-    parser.removeErrorListeners();
-    parser.addErrorListener(new ErrorListener());
-    parser.transformation();
-    return ret;
+    StellarCompiler.Expression expression = null;
+    try {
+      expression = expressionCache.get(rule, () -> compile(rule));
+    } catch (ExecutionException e) {
+      throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e);
+    }
+    return expression.variablesUsed;
   }
 
   /**
@@ -104,6 +115,24 @@ public class BaseStellarProcessor<T> {
    * @return The value of the evaluated Stellar expression, {@code rule}.
    */
   public T parse(final String rule, final VariableResolver variableResolver, final FunctionResolver functionResolver, final Context context) {
+    StellarCompiler.Expression expression = null;
+    if (rule == null || isEmpty(rule.trim())) {
+      return null;
+    }
+    try {
+      expression = expressionCache.get(rule, () -> compile(rule));
+    } catch (ExecutionException|UncheckedExecutionException e) {
+      throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e);
+    }
+    return clazz.cast(expression.apply(new StellarCompiler.ExpressionState(context, functionResolver, variableResolver)));
+  }
+
+  /**
+   * Parses and evaluates the given Stellar expression, {@code rule}.
+   * @param rule The Stellar expression to parse and evaluate.
+   * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers.
+   */
+  public StellarCompiler.Expression compile(final String rule) {
     if (rule == null || isEmpty(rule.trim())) {
       return null;
     }
@@ -115,7 +144,7 @@ public class BaseStellarProcessor<T> {
     TokenStream tokens = new CommonTokenStream(lexer);
     StellarParser parser = new StellarParser(tokens);
 
-    StellarCompiler treeBuilder = new StellarCompiler(variableResolver, functionResolver, context, new Stack<>(),
+    StellarCompiler treeBuilder = new StellarCompiler(
         ArithmeticEvaluator.INSTANCE,
         NumberLiteralEvaluator.INSTANCE,
         ComparisonExpressionWithOperatorEvaluator.INSTANCE
@@ -124,7 +153,7 @@ public class BaseStellarProcessor<T> {
     parser.removeErrorListeners();
     parser.addErrorListener(new ErrorListener());
     parser.transformation();
-    return clazz.cast(treeBuilder.getResult());
+    return treeBuilder.getExpression();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
index 4af299e..1c37009 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
@@ -6,68 +6,114 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.metron.common.stellar;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableSet;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.FunctionMarker;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunction;
 import org.apache.metron.common.dsl.Token;
 import org.apache.metron.common.dsl.VariableResolver;
+import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
 import org.apache.metron.common.stellar.evaluators.ArithmeticEvaluator;
 import org.apache.metron.common.stellar.evaluators.ComparisonExpressionWithOperatorEvaluator;
 import org.apache.metron.common.stellar.evaluators.NumberLiteralEvaluator;
 import org.apache.metron.common.stellar.generated.StellarBaseListener;
 import org.apache.metron.common.stellar.generated.StellarParser;
+import com.google.common.base.Joiner;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.metron.common.dsl.FunctionMarker;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunction;
 import org.apache.metron.common.utils.ConversionUtils;
 
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
+import java.util.*;
 
 import static java.lang.String.format;
 
 public class StellarCompiler extends StellarBaseListener {
-
-  private final Context context;
-  private final Stack<Token<?>> tokenStack;
-  private final FunctionResolver functionResolver;
-  private final VariableResolver variableResolver;
-  private Throwable actualException;
+  private Expression expression;
   private final ArithmeticEvaluator arithmeticEvaluator;
   private final NumberLiteralEvaluator numberLiteralEvaluator;
   private final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator;
 
-  public StellarCompiler(final VariableResolver variableResolver,
-                         final FunctionResolver functionResolver,
-                         final Context context,
-                         final Stack<Token<?>> tokenStack,
-                         final ArithmeticEvaluator arithmeticEvaluator,
-                         final NumberLiteralEvaluator numberLiteralEvaluator,
-                         final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator) {
-    this.variableResolver = variableResolver;
-    this.functionResolver = functionResolver;
-    this.context = context;
-    this.tokenStack = tokenStack;
+  public static class ExpressionState {
+    Context context;
+    FunctionResolver functionResolver;
+    VariableResolver variableResolver;
+    public ExpressionState(Context context
+              , FunctionResolver functionResolver
+              , VariableResolver variableResolver
+                          ) {
+      this.context = context;
+      this.variableResolver = variableResolver;
+      this.functionResolver = functionResolver;
+    }
+  }
+
+  public static class Expression {
+    final Deque<Token<?>> tokenDeque;
+    final Set<String> variablesUsed;
+    Expression(Deque<Token<?>> tokenDeque) {
+      this.tokenDeque = tokenDeque;
+      this.variablesUsed = new HashSet<>();
+    }
+
+    public Object apply(ExpressionState state) {
+      Deque<Token<?>> instanceDeque = new ArrayDeque<>();
+      for(Iterator<Token<?>> it = tokenDeque.descendingIterator();it.hasNext();) {
+        Token<?> token = it.next();
+        if(token.getUnderlyingType() == DeferredFunction.class) {
+          DeferredFunction func = (DeferredFunction) token.getValue();
+          func.apply(instanceDeque, state);
+        }
+        else {
+          instanceDeque.push(token);
+        }
+      }
+
+      if (instanceDeque.isEmpty()) {
+        throw new ParseException("Invalid predicate: Empty stack.");
+      }
+      Token<?> token = instanceDeque.pop();
+      if (instanceDeque.isEmpty()) {
+        return token.getValue();
+      }
+      if (instanceDeque.isEmpty()) {
+        throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(instanceDeque));
+      } else {
+        throw new ParseException("Invalid parse, found " + token);
+      }
+    }
+  }
+
+  interface DeferredFunction {
+    void apply( Deque<Token<?>> tokenDeque
+              , ExpressionState state
+              );
+  }
+
+  public StellarCompiler(
+          final ArithmeticEvaluator arithmeticEvaluator,
+          final NumberLiteralEvaluator numberLiteralEvaluator,
+          final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator
+  ){
+    this(new Expression(new ArrayDeque<>()), arithmeticEvaluator, numberLiteralEvaluator, comparisonExpressionWithOperatorEvaluator);
+  }
+
+  public StellarCompiler(
+          final Expression expression,
+          final ArithmeticEvaluator arithmeticEvaluator,
+          final NumberLiteralEvaluator numberLiteralEvaluator,
+          final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator
+  ){
+    this.expression = expression;
     this.arithmeticEvaluator = arithmeticEvaluator;
     this.numberLiteralEvaluator = numberLiteralEvaluator;
     this.comparisonExpressionWithOperatorEvaluator = comparisonExpressionWithOperatorEvaluator;
@@ -75,7 +121,7 @@ public class StellarCompiler extends StellarBaseListener {
 
   @Override
   public void enterTransformation(StellarParser.TransformationContext ctx) {
-    tokenStack.clear();
+    expression.tokenDeque.clear();
   }
 
   private boolean handleIn(final Token<?> left, final Token<?> right) {
@@ -107,50 +153,60 @@ public class StellarCompiler extends StellarBaseListener {
 
   @Override
   public void exitNullConst(StellarParser.NullConstContext ctx) {
-    tokenStack.push(new Token<>(null, Object.class));
+    expression.tokenDeque.push(new Token<>(null, Object.class));
   }
 
   @Override
   public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) {
-    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair();
-    tokenStack.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.addition(), p));
+    expression.tokenDeque.push(new Token<>((tokenDeque, state) -> {
+      Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque);
+      tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.addition(), p));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) {
-    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair();
-    tokenStack.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.subtraction(), p));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque);
+    tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.subtraction(), p));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) {
-    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair();
-    tokenStack.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.division(), p));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque);
+    tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.division(), p));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) {
-    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair();
-    tokenStack.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.multiplication(), p));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque);
+    tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.multiplication(), p));
+    }, DeferredFunction.class));
   }
 
   @SuppressWarnings("unchecked")
-  private Pair<Token<? extends Number>, Token<? extends Number>> getArithExpressionPair() {
-    Token<? extends Number> right = (Token<? extends Number>) popStack();
-    Token<? extends Number> left = (Token<? extends Number>) popStack();
+  private Pair<Token<? extends Number>, Token<? extends Number>> getArithExpressionPair(Deque<Token<?>> tokenDeque) {
+    Token<? extends Number> right = (Token<? extends Number>) popDeque(tokenDeque);
+    Token<? extends Number> left = (Token<? extends Number>) popDeque(tokenDeque);
     return Pair.of(left, right);
   }
 
   private void handleConditional() {
-    Token<?> elseExpr = popStack();
-    Token<?> thenExpr = popStack();
-    Token<?> ifExpr = popStack();
-    @SuppressWarnings("unchecked") boolean b = ((Token<Boolean>) ifExpr).getValue();
-    if (b) {
-      tokenStack.push(thenExpr);
-    } else {
-      tokenStack.push(elseExpr);
-    }
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      Token<?> elseExpr = popDeque(tokenDeque);
+      Token<?> thenExpr = popDeque(tokenDeque);
+      Token<?> ifExpr = popDeque(tokenDeque);
+      @SuppressWarnings("unchecked") boolean b = ((Token<Boolean>) ifExpr).getValue();
+      if (b) {
+        tokenDeque.push(thenExpr);
+      } else {
+        tokenDeque.push(elseExpr);
+      }
+    }, DeferredFunction.class));
   }
 
   @Override
@@ -165,67 +221,80 @@ public class StellarCompiler extends StellarBaseListener {
 
   @Override
   public void exitInExpressionStatement(StellarParser.InExpressionStatementContext ctx) {
-    Token<?> left = popStack();
-    Token<?> right = popStack();
-    tokenStack.push(new Token<>(handleIn(left, right), Boolean.class));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Token<?> left = popDeque(tokenDeque);
+    Token<?> right = popDeque(tokenDeque);
+    tokenDeque.push(new Token<>(handleIn(left, right), Boolean.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitNInExpressionStatement(StellarParser.NInExpressionStatementContext ctx) {
-    Token<?> left = popStack();
-    Token<?> right = popStack();
-    tokenStack.push(new Token<>(!handleIn(left, right), Boolean.class));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Token<?> left = popDeque(tokenDeque);
+    Token<?> right = popDeque(tokenDeque);
+    tokenDeque.push(new Token<>(!handleIn(left, right), Boolean.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitNotFunc(StellarParser.NotFuncContext ctx) {
-    Token<Boolean> arg = (Token<Boolean>) popStack();
-    tokenStack.push(new Token<>(!arg.getValue(), Boolean.class));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Token<Boolean> arg = (Token<Boolean>) popDeque(tokenDeque);
+    tokenDeque.push(new Token<>(!arg.getValue(), Boolean.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitVariable(StellarParser.VariableContext ctx) {
-    tokenStack.push(new Token<>(variableResolver.resolve(ctx.getText()), Object.class));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      tokenDeque.push(new Token<>(state.variableResolver.resolve(ctx.getText()), Object.class));
+    }, DeferredFunction.class));
+    expression.variablesUsed.add(ctx.getText());
   }
 
   @Override
   public void exitStringLiteral(StellarParser.StringLiteralContext ctx) {
-    tokenStack.push(new Token<>(ctx.getText().substring(1, ctx.getText().length() - 1), String.class));
+    expression.tokenDeque.push(new Token<>(ctx.getText().substring(1, ctx.getText().length() - 1), String.class));
   }
 
   @Override
   public void exitIntLiteral(StellarParser.IntLiteralContext ctx) {
-    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+    expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override
   public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) {
-    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+    expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override
   public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) {
-    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+    expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override
   public void exitLongLiteral(StellarParser.LongLiteralContext ctx) {
-    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+    expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override
   public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) {
-    Token<?> left = popStack();
-    Token<?> right = popStack();
-    tokenStack.push(new Token<>(booleanOp(left, right, (l, r) -> l && r, "&&"), Boolean.class));
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Token<?> left = popDeque(tokenDeque);
+    Token<?> right = popDeque(tokenDeque);
+    tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l && r, "&&"), Boolean.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) {
-    Token<?> left = popStack();
-    Token<?> right = popStack();
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+    Token<?> left = popDeque(tokenDeque);
+    Token<?> right = popDeque(tokenDeque);
 
-    tokenStack.push(new Token<>(booleanOp(left, right, (l, r) -> l || r, "||"), Boolean.class));
+    tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l || r, "||"), Boolean.class));
+    }, DeferredFunction.class));
   }
 
   @Override
@@ -241,7 +310,7 @@ public class StellarCompiler extends StellarBaseListener {
       default:
         throw new ParseException("Unable to process " + ctx.getText() + " as a boolean constant");
     }
-    tokenStack.push(new Token<>(b, Boolean.class));
+    expression.tokenDeque.push(new Token<>(b, Boolean.class));
   }
 
   private boolean booleanOp(final Token<?> left, final Token<?> right, final BooleanOp op, final String opName) {
@@ -256,20 +325,17 @@ public class StellarCompiler extends StellarBaseListener {
   @Override
   public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) {
 
-    // resolve and initialize the function
-    String functionName = ctx.getChild(0).getText();
-    StellarFunction function = resolveFunction(functionName);
-    initializeFunction(function, functionName);
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      // resolve and initialize the function
+      String functionName = ctx.getChild(0).getText();
+      StellarFunction function = resolveFunction(state.functionResolver, functionName);
+      initializeFunction(state.context, function, functionName);
 
-    // fetch the args, execute, and push result onto the stack
-    List<Object> args = getFunctionArguments(popStack());
-    try {
-      Object result = function.apply(args, context);
-      tokenStack.push(new Token<>(result, Object.class));
-    }
-    catch(Throwable t) {
-      actualException = t;
-    }
+      // fetch the args, execute, and push result onto the stack
+      List<Object> args = getFunctionArguments(popDeque(tokenDeque));
+      Object result = function.apply(args, state.context);
+      tokenDeque.push(new Token<>(result, Object.class));
+    }, DeferredFunction.class));
   }
 
   /**
@@ -292,7 +358,7 @@ public class StellarCompiler extends StellarBaseListener {
    * @param funcName
    * @return
    */
-  private StellarFunction resolveFunction(String funcName) {
+  private StellarFunction resolveFunction(FunctionResolver functionResolver, String funcName) {
     try {
       return functionResolver.apply(funcName);
 
@@ -308,7 +374,7 @@ public class StellarCompiler extends StellarBaseListener {
    * @param function The function to initialize.
    * @param functionName The name of the functions.
    */
-  private void initializeFunction(StellarFunction function, String functionName) {
+  private void initializeFunction(Context context, StellarFunction function, String functionName) {
     try {
       if (!function.isInitialized()) {
         function.initialize(context);
@@ -321,104 +387,101 @@ public class StellarCompiler extends StellarBaseListener {
 
   @Override
   public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) {
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      String variable = ctx.getChild(2).getText();
+      boolean exists = state.variableResolver.resolve(variable) != null;
+      tokenDeque.push(new Token<>(exists, Boolean.class));
+    }, DeferredFunction.class));
     String variable = ctx.getChild(2).getText();
-    boolean exists = variableResolver.resolve(variable) != null;
-    tokenStack.push(new Token<>(exists, Boolean.class));
+    expression.variablesUsed.add(variable);
   }
 
   @Override
   public void enterFunc_args(StellarParser.Func_argsContext ctx) {
-    tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
+    expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
   }
 
   @Override
   public void exitFunc_args(StellarParser.Func_argsContext ctx) {
-    LinkedList<Object> args = new LinkedList<>();
-    while (true) {
-      Token<?> token = popStack();
-      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
-        break;
-      } else {
-        args.addFirst(token.getValue());
+    expression.tokenDeque.push(new Token<>((tokenDeque, state) -> {
+      LinkedList<Object> args = new LinkedList<>();
+      while (true) {
+        Token<?> token = popDeque(tokenDeque);
+        if (token.getUnderlyingType().equals(FunctionMarker.class)) {
+          break;
+        } else {
+          args.addFirst(token.getValue());
+        }
       }
-    }
-    tokenStack.push(new Token<>(args, List.class));
+      tokenDeque.push(new Token<>(args, List.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void enterMap_entity(StellarParser.Map_entityContext ctx) {
-    tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
+    expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
   }
 
   @Override
   public void exitMap_entity(StellarParser.Map_entityContext ctx) {
-    HashMap<String, Object> args = new HashMap<>();
-    Object value = null;
-    for (int i = 0; true; i++) {
-      Token<?> token = popStack();
-      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
-        break;
-      } else {
-        if (i % 2 == 0) {
-          value = token.getValue();
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      HashMap<String, Object> args = new HashMap<>();
+      Object value = null;
+      for (int i = 0; true; i++) {
+        Token<?> token = popDeque(tokenDeque);
+        if (token.getUnderlyingType().equals(FunctionMarker.class)) {
+          break;
         } else {
-          args.put(token.getValue() + "", value);
+          if (i % 2 == 0) {
+            value = token.getValue();
+          } else {
+            args.put(token.getValue() + "", value);
+          }
         }
       }
-    }
-    tokenStack.push(new Token<>(args, Map.class));
+      tokenDeque.push(new Token<>(args, Map.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitList_entity(StellarParser.List_entityContext ctx) {
-    LinkedList<Object> args = new LinkedList<>();
-    while (true) {
-      Token<?> token = popStack();
-      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
-        break;
-      } else {
-        args.addFirst(token.getValue());
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      LinkedList<Object> args = new LinkedList<>();
+      while (true) {
+        Token<?> token = popDeque(tokenDeque);
+        if (token.getUnderlyingType().equals(FunctionMarker.class)) {
+          break;
+        } else {
+          args.addFirst(token.getValue());
+        }
       }
-    }
-    tokenStack.push(new Token<>(args, List.class));
+      tokenDeque.push(new Token<>(args, List.class));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) {
-    StellarParser.Comp_operatorContext op = ctx.comp_operator();
-    Token<?> right = popStack();
-    Token<?> left = popStack();
+    expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> {
+      StellarParser.Comp_operatorContext op = ctx.comp_operator();
+      Token<?> right = popDeque(tokenDeque);
+      Token<?> left = popDeque(tokenDeque);
 
-    tokenStack.push(comparisonExpressionWithOperatorEvaluator.evaluate(left, right, (StellarParser.ComparisonOpContext) op));
+      tokenDeque.push(comparisonExpressionWithOperatorEvaluator.evaluate(left, right, (StellarParser.ComparisonOpContext) op));
+    }, DeferredFunction.class));
   }
 
   @Override
   public void enterList_entity(StellarParser.List_entityContext ctx) {
-    tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
+    expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class));
   }
 
-  private Token<?> popStack() {
-    if (tokenStack.empty()) {
+  private Token<?> popDeque(Deque<Token<?>> tokenDeque) {
+    if (tokenDeque.isEmpty()) {
       throw new ParseException("Unable to pop an empty stack");
     }
-    return tokenStack.pop();
+    return tokenDeque.pop();
   }
 
-  public Object getResult() throws ParseException {
-    if (actualException != null) {
-      throw new ParseException("Unable to execute: " + actualException.getMessage(), actualException);
-    }
-    if (tokenStack.empty()) {
-      throw new ParseException("Invalid predicate: Empty stack.");
-    }
-    Token<?> token = popStack();
-    if (tokenStack.empty()) {
-      return token.getValue();
-    }
-    if (tokenStack.empty()) {
-      throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(tokenStack));
-    } else {
-      throw new ParseException("Invalid parse, found " + token);
-    }
-  }
+  public Expression getExpression() {return expression;}
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java
index 1701879..581c9c1 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarPredicateProcessor.java
@@ -23,6 +23,8 @@ import org.apache.metron.common.dsl.Context;
 import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
 import org.apache.metron.common.dsl.VariableResolver;
 
+import java.util.concurrent.TimeUnit;
+
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 
 /**
@@ -38,6 +40,9 @@ public class StellarPredicateProcessor extends BaseStellarProcessor<Boolean> {
     super(Boolean.class);
   }
 
+  public StellarPredicateProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) {
+    super(Boolean.class, cacheSize, expiryTime, expiryUnit);
+  }
   @Override
   public Boolean parse( String rule
                       , VariableResolver variableResolver

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarProcessor.java
index 380aedf..ae6238b 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarProcessor.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarProcessor.java
@@ -18,6 +18,8 @@
 
 package org.apache.metron.common.stellar;
 
+import java.util.concurrent.TimeUnit;
+
 /**
  * The Stellar Processor is intended to allow for general transformations using the Stellar
  * domain specific language.  In contrast to the StellarPredicateProcessor where
@@ -30,4 +32,8 @@ public class StellarProcessor extends BaseStellarProcessor<Object> {
   public StellarProcessor() {
     super(Object.class);
   }
+
+  public StellarProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) {
+    super(Object.class, cacheSize, expiryTime, expiryUnit);
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/Microbenchmark.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/Microbenchmark.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/Microbenchmark.java
new file mode 100644
index 0000000..bce5014
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/Microbenchmark.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.stellar.benchmark;
+
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.VariableResolver;
+import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.common.stellar.StellarProcessor;
+
+import java.util.function.Consumer;
+
+public class Microbenchmark {
+
+  public static class StellarStatement {
+    String expression;
+    VariableResolver variableResolver;
+    FunctionResolver functionResolver;
+    Context context;
+  }
+
+  public static DescriptiveStatistics run(StellarStatement statement, int warmupRounds, int benchmarkRounds )
+  {
+    run(warmupRounds, statement, ts -> {});
+    final DescriptiveStatistics stats = new DescriptiveStatistics();
+    run(benchmarkRounds, statement, ts -> { stats.addValue(ts);});
+    return stats;
+  }
+
+  private static void run(int numTimes, StellarStatement statement, Consumer<Long> func) {
+    StellarProcessor processor = new StellarProcessor();
+    for(int i = 0;i < numTimes;++i) {
+      long start = System.nanoTime();
+      processor.parse(statement.expression, statement.variableResolver, statement.functionResolver, statement.context);
+      func.accept((System.nanoTime() - start)/1000);
+    }
+  }
+
+  public static String describe(DescriptiveStatistics stats, Double[] percentiles){
+    StringBuilder sb = new StringBuilder();
+    sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n",
+            (long)stats.getMean(),
+            (long)stats.getStandardDeviation(), stats.getN() ));
+    sb.append("\tMin - " + (long)stats.getMin() + "\n");
+    for(double pctile : percentiles) {
+      sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n");
+    }
+    sb.append("\tMax - " + (long)stats.getMax());
+    return sb.toString();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/StellarMicrobenchmark.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/StellarMicrobenchmark.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/StellarMicrobenchmark.java
new file mode 100644
index 0000000..d674210
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/benchmark/StellarMicrobenchmark.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.stellar.benchmark;
+
+import com.clearspring.analytics.util.Lists;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.io.Files;
+import org.apache.commons.cli.*;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.MapVariableResolver;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.utils.cli.OptionHandler;
+import scala.testing.Benchmark;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.charset.Charset;
+import java.util.*;
+
+public class StellarMicrobenchmark {
+
+  public static int DEFAULT_WARMUP = 100;
+  public static int DEFAULT_NUM_TIMES = 1000;
+  public static Double[] DEFAULT_PERCENTILES = new Double[] {
+    50d, 75d, 95d, 99d
+  };
+
+  enum BenchmarkOptions {
+    HELP("h", new OptionHandler<BenchmarkOptions>() {
+
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        return new Option(s, "help", false, "Generate Help screen");
+      }
+    }),
+    WARMUP("w", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "warmup", true, "Number of times for warmup per expression. Default: " + DEFAULT_WARMUP);
+        o.setArgName("NUM");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    }),
+    PERCENTILES("p", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "percentiles", true
+                             , "Percentiles to calculate per run. Default: " + Joiner.on(",").join(Arrays.asList(DEFAULT_PERCENTILES))
+                             );
+        o.setArgName("NUM");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    }),
+    NUM_TIMES("n", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "num_times", true, "Number of times to run per expression (after warmup). Default: " + DEFAULT_NUM_TIMES);
+        o.setArgName("NUM");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    }),
+    EXPRESSIONS("e", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "expressions", true, "Stellar expressions");
+        o.setArgName("FILE");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    }),
+    VARIABLES("v", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "variables", true, "File containing a JSON Map of variables to use");
+        o.setArgName("FILE");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    }),
+    OUTPUT("o", new OptionHandler<BenchmarkOptions>() {
+      @Nullable
+      @Override
+      public Option apply(@Nullable String s) {
+        Option o = new Option(s, "output", true, "File to write output.");
+        o.setArgName("FILE");
+        o.setRequired(false);
+        return o;
+      }
+
+      @Override
+      public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) {
+        return Optional.ofNullable(option.get(cli).trim());
+      }
+    })
+    ;
+    ;
+    Option option;
+    String shortCode;
+    OptionHandler<BenchmarkOptions> handler;
+    BenchmarkOptions(String shortCode, OptionHandler<BenchmarkOptions> optionHandler) {
+      this.shortCode = shortCode;
+      this.handler = optionHandler;
+      this.option = optionHandler.apply(shortCode);
+    }
+
+    public boolean has(CommandLine cli) {
+      return cli.hasOption(shortCode);
+    }
+
+    public String get(CommandLine cli) {
+      return cli.getOptionValue(shortCode);
+    }
+
+    public static CommandLine parse(CommandLineParser parser, String[] args) {
+      try {
+        CommandLine cli = parser.parse(getOptions(), args);
+        if(HELP.has(cli)) {
+          printHelp();
+          System.exit(0);
+        }
+        return cli;
+      } catch (org.apache.commons.cli.ParseException e) {
+        System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
+        e.printStackTrace(System.err);
+        printHelp();
+        System.exit(-1);
+        return null;
+      }
+    }
+
+    public static EnumMap<BenchmarkOptions, Optional<Object>> createConfig(CommandLine cli) {
+      EnumMap<BenchmarkOptions, Optional<Object> > ret = new EnumMap<>(BenchmarkOptions.class);
+      for(BenchmarkOptions option : values()) {
+        ret.put(option, option.handler.getValue(option, cli));
+      }
+      return ret;
+    }
+
+    public static void printHelp() {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp( "StellarBenchmark", getOptions());
+    }
+
+    public static Options getOptions() {
+      Options ret = new Options();
+      for(BenchmarkOptions o : BenchmarkOptions.values()) {
+        ret.addOption(o.option);
+      }
+      return ret;
+    }
+  }
+
+  public static void main(String... argv) throws IOException {
+    CommandLine cli = BenchmarkOptions.parse(new PosixParser(), argv);
+    if(!BenchmarkOptions.EXPRESSIONS.has(cli)) {
+      throw new IllegalStateException("You must at least specify an expressions file.");
+    }
+    File expressionsFile = new File(BenchmarkOptions.EXPRESSIONS.get(cli));
+    Optional<File> variablesFile = Optional.ofNullable(!BenchmarkOptions.VARIABLES.has(cli)
+                                                      ?null
+                                                      :new File(BenchmarkOptions.VARIABLES.get(cli))
+                                                      );
+    Optional<File> output = Optional.ofNullable(!BenchmarkOptions.OUTPUT.has(cli)
+                                             ?null
+                                             :new File(BenchmarkOptions.OUTPUT.get(cli))
+                                             );
+    List<String> lines = Files.readLines(expressionsFile, Charset.defaultCharset());
+    Map<String, Object> variables = new HashMap<>();
+    if(variablesFile.isPresent()) {
+      variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), new TypeReference<Map<String, Object>>() {
+      });
+    }
+    int numTimes = DEFAULT_NUM_TIMES;
+    if(BenchmarkOptions.NUM_TIMES.has(cli)) {
+      numTimes = Integer.parseInt(BenchmarkOptions.NUM_TIMES.get(cli));
+    }
+    int warmup = DEFAULT_WARMUP;
+    if(BenchmarkOptions.WARMUP.has(cli)) {
+      warmup = Integer.parseInt(BenchmarkOptions.WARMUP.get(cli));
+    }
+    Double[] percentiles = DEFAULT_PERCENTILES;
+    if(BenchmarkOptions.PERCENTILES.has(cli)) {
+      List<Double> percentileList = new ArrayList<>();
+      for(String token : Splitter.on(",").split(BenchmarkOptions.PERCENTILES.get(cli))) {
+        if(token.trim().isEmpty()) {
+          continue;
+        }
+        Double d = Double.parseDouble(token.trim());
+        percentileList.add(d);
+      }
+      percentiles = (Double[])percentileList.toArray();
+    }
+    PrintWriter out = new PrintWriter(System.out);
+    if(output.isPresent()) {
+      out = new PrintWriter(output.get());
+    }
+    for(String statement : lines) {
+      if(statement.trim().startsWith("#") || statement.trim().isEmpty()) {
+        continue;
+      }
+      Microbenchmark.StellarStatement s = new Microbenchmark.StellarStatement();
+      s.context = Context.EMPTY_CONTEXT();
+      s.expression = statement;
+      s.functionResolver = StellarFunctions.FUNCTION_RESOLVER();
+      s.variableResolver = new MapVariableResolver(variables);
+      DescriptiveStatistics stats = Microbenchmark.run(s, warmup, numTimes);
+      out.println("Expression: " + statement);
+      out.println(Microbenchmark.describe(stats, percentiles));
+    }
+    if(argv.length > 2) {
+      out.close();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarCompilerTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarCompilerTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarCompilerTest.java
deleted file mode 100644
index 3365643..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarCompilerTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.stellar;
-
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.Token;
-import org.apache.metron.common.dsl.VariableResolver;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.stellar.evaluators.ArithmeticEvaluator;
-import org.apache.metron.common.stellar.evaluators.ComparisonExpressionWithOperatorEvaluator;
-import org.apache.metron.common.stellar.evaluators.NumberLiteralEvaluator;
-import org.apache.metron.common.stellar.generated.StellarParser;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.Stack;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.*;
-
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ArithmeticEvaluator.class, NumberLiteralEvaluator.class, ComparisonExpressionWithOperatorEvaluator.class})
-public class StellarCompilerTest {
-  VariableResolver variableResolver;
-  FunctionResolver functionResolver;
-  Context context;
-  Stack<Token<?>> tokenStack;
-  ArithmeticEvaluator arithmeticEvaluator;
-  NumberLiteralEvaluator numberLiteralEvaluator;
-  ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator;
-  StellarCompiler compiler;
-
-  @SuppressWarnings("unchecked")
-  @Before
-  public void setUp() throws Exception {
-    variableResolver = mock(VariableResolver.class);
-    functionResolver = mock(FunctionResolver.class);
-    context = mock(Context.class);
-    tokenStack = mock(Stack.class);
-    arithmeticEvaluator = mock(ArithmeticEvaluator.class);
-    numberLiteralEvaluator = mock(NumberLiteralEvaluator.class);
-    comparisonExpressionWithOperatorEvaluator = mock(ComparisonExpressionWithOperatorEvaluator.class);
-
-    compiler = new StellarCompiler(variableResolver, functionResolver, context, tokenStack, arithmeticEvaluator, numberLiteralEvaluator, comparisonExpressionWithOperatorEvaluator);
-  }
-
-  @Test
-  public void exitIntLiteralShouldProperlyParseStringsAsIntegers() throws Exception {
-    StellarParser.IntLiteralContext ctx = mock(StellarParser.IntLiteralContext.class);
-    when(ctx.getText()).thenReturn("1000");
-
-    compiler.exitIntLiteral(ctx);
-
-    verify(numberLiteralEvaluator).evaluate(ctx);
-    verify(tokenStack).push(any(Token.class));
-    verifyNoMoreInteractions(tokenStack);
-    verifyZeroInteractions(variableResolver);
-    verifyZeroInteractions(functionResolver);
-    verifyZeroInteractions(context);
-    verifyZeroInteractions(arithmeticEvaluator);
-    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
-  }
-
-  @Test
-  public void exitDoubleLiteralShouldProperlyParseStringsAsDoubles() throws Exception {
-    StellarParser.DoubleLiteralContext ctx = mock(StellarParser.DoubleLiteralContext.class);
-    when(ctx.getText()).thenReturn("1000D");
-
-    compiler.exitDoubleLiteral(ctx);
-
-    verify(numberLiteralEvaluator).evaluate(ctx);
-    verify(tokenStack).push(any(Token.class));
-    verifyNoMoreInteractions(tokenStack);
-    verifyZeroInteractions(variableResolver);
-    verifyZeroInteractions(functionResolver);
-    verifyZeroInteractions(context);
-    verifyZeroInteractions(arithmeticEvaluator);
-    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
-  }
-
-  @Test
-  public void exitFloatLiteralShouldProperlyParseStringsAsFloats() throws Exception {
-    StellarParser.FloatLiteralContext ctx = mock(StellarParser.FloatLiteralContext.class);
-    when(ctx.getText()).thenReturn("1000f");
-
-    compiler.exitFloatLiteral(ctx);
-
-    verify(numberLiteralEvaluator).evaluate(ctx);
-    verify(tokenStack).push(any(Token.class));
-    verifyNoMoreInteractions(tokenStack);
-    verifyZeroInteractions(variableResolver);
-    verifyZeroInteractions(functionResolver);
-    verifyZeroInteractions(context);
-    verifyZeroInteractions(arithmeticEvaluator);
-    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
-  }
-
-  @Test
-  public void exitLongLiteralShouldProperlyParseStringsAsLongs() throws Exception {
-    StellarParser.LongLiteralContext ctx = mock(StellarParser.LongLiteralContext.class);
-    when(ctx.getText()).thenReturn("1000l");
-
-    compiler.exitLongLiteral(ctx);
-
-    verify(numberLiteralEvaluator).evaluate(ctx);
-    verify(tokenStack).push(any(Token.class));
-    verifyNoMoreInteractions(tokenStack);
-    verifyZeroInteractions(variableResolver);
-    verifyZeroInteractions(functionResolver);
-    verifyZeroInteractions(context);
-    verifyZeroInteractions(arithmeticEvaluator);
-    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
-  }
-
-  @Test
-  public void properlyCompareTwoNumbers() throws Exception {
-    StellarParser.ComparisonExpressionWithOperatorContext ctx = mock(StellarParser.ComparisonExpressionWithOperatorContext.class);
-    StellarParser.ComparisonOpContext mockOp = mock(StellarParser.ComparisonOpContext.class);
-    when(ctx.comp_operator()).thenReturn(mockOp);
-
-    compiler.exitComparisonExpressionWithOperator(ctx);
-
-    verify(comparisonExpressionWithOperatorEvaluator).evaluate(any(Token.class), any(Token.class), eq(mockOp));
-    verify(tokenStack, times(2)).pop();
-    verify(tokenStack).push(any());
-    verify(tokenStack, times(2)).empty();
-    verifyNoMoreInteractions(tokenStack);
-    verifyZeroInteractions(numberLiteralEvaluator);
-    verifyZeroInteractions(variableResolver);
-    verifyZeroInteractions(functionResolver);
-    verifyZeroInteractions(context);
-    verifyZeroInteractions(arithmeticEvaluator);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarInterpreterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarInterpreterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarInterpreterTest.java
new file mode 100644
index 0000000..b25f0a7
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarInterpreterTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.stellar;
+
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.Token;
+import org.apache.metron.common.dsl.VariableResolver;
+import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.common.stellar.evaluators.ArithmeticEvaluator;
+import org.apache.metron.common.stellar.evaluators.ComparisonExpressionWithOperatorEvaluator;
+import org.apache.metron.common.stellar.evaluators.NumberLiteralEvaluator;
+import org.apache.metron.common.stellar.generated.StellarParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.verification.VerificationMode;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Stack;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.*;
+
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Deque.class, ArithmeticEvaluator.class, NumberLiteralEvaluator.class, ComparisonExpressionWithOperatorEvaluator.class})
+public class StellarInterpreterTest {
+  VariableResolver variableResolver;
+  FunctionResolver functionResolver;
+  Context context;
+  Deque<Token<?>> tokenStack;
+  ArithmeticEvaluator arithmeticEvaluator;
+  NumberLiteralEvaluator numberLiteralEvaluator;
+  ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator;
+  StellarCompiler compiler;
+  StellarCompiler.Expression expression;
+
+  @SuppressWarnings("unchecked")
+  @Before
+  public void setUp() throws Exception {
+    variableResolver = mock(VariableResolver.class);
+    functionResolver = mock(FunctionResolver.class);
+    context = mock(Context.class);
+    tokenStack = new ArrayDeque<>();
+    arithmeticEvaluator = mock(ArithmeticEvaluator.class);
+    numberLiteralEvaluator = mock(NumberLiteralEvaluator.class);
+    comparisonExpressionWithOperatorEvaluator = mock(ComparisonExpressionWithOperatorEvaluator.class);
+    expression = new StellarCompiler.Expression(tokenStack);
+    compiler = new StellarCompiler(expression, arithmeticEvaluator, numberLiteralEvaluator, comparisonExpressionWithOperatorEvaluator);
+  }
+
+  @Test
+  public void exitIntLiteralShouldProperlyParseStringsAsIntegers() throws Exception {
+    StellarParser.IntLiteralContext ctx = mock(StellarParser.IntLiteralContext.class);
+    Token result = mock(Token.class);
+    when(ctx.getText()).thenReturn("1000");
+    when(numberLiteralEvaluator.evaluate(ctx)).thenReturn(result);
+    compiler.exitIntLiteral(ctx);
+    verify(numberLiteralEvaluator).evaluate(ctx);
+    Assert.assertEquals(1, tokenStack.size());
+    Assert.assertEquals(tokenStack.getFirst(), result);
+    verifyZeroInteractions(variableResolver);
+    verifyZeroInteractions(functionResolver);
+    verifyZeroInteractions(context);
+    verifyZeroInteractions(arithmeticEvaluator);
+    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
+  }
+
+  @Test
+  public void exitDoubleLiteralShouldProperlyParseStringsAsDoubles() throws Exception {
+    StellarParser.DoubleLiteralContext ctx = mock(StellarParser.DoubleLiteralContext.class);
+    Token result = mock(Token.class);
+    when(numberLiteralEvaluator.evaluate(ctx)).thenReturn(result);
+    when(ctx.getText()).thenReturn("1000D");
+
+    compiler.exitDoubleLiteral(ctx);
+
+    verify(numberLiteralEvaluator).evaluate(ctx);
+    Assert.assertEquals(1, tokenStack.size());
+    Assert.assertEquals(tokenStack.getFirst(), result);
+    verifyZeroInteractions(variableResolver);
+    verifyZeroInteractions(functionResolver);
+    verifyZeroInteractions(context);
+    verifyZeroInteractions(arithmeticEvaluator);
+    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
+  }
+
+  @Test
+  public void exitFloatLiteralShouldProperlyParseStringsAsFloats() throws Exception {
+    StellarParser.FloatLiteralContext ctx = mock(StellarParser.FloatLiteralContext.class);
+    when(ctx.getText()).thenReturn("1000f");
+    Token result = mock(Token.class);
+    when(numberLiteralEvaluator.evaluate(ctx)).thenReturn(result);
+
+    compiler.exitFloatLiteral(ctx);
+
+    verify(numberLiteralEvaluator).evaluate(ctx);
+    Assert.assertEquals(1, tokenStack.size());
+    Assert.assertEquals(tokenStack.getFirst(), result);
+    verifyZeroInteractions(variableResolver);
+    verifyZeroInteractions(functionResolver);
+    verifyZeroInteractions(context);
+    verifyZeroInteractions(arithmeticEvaluator);
+    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
+  }
+
+  @Test
+  public void exitLongLiteralShouldProperlyParseStringsAsLongs() throws Exception {
+    StellarParser.LongLiteralContext ctx = mock(StellarParser.LongLiteralContext.class);
+    when(ctx.getText()).thenReturn("1000l");
+    Token result = mock(Token.class);
+    when(numberLiteralEvaluator.evaluate(ctx)).thenReturn(result);
+
+    compiler.exitLongLiteral(ctx);
+
+    verify(numberLiteralEvaluator).evaluate(ctx);
+    Assert.assertEquals(1, tokenStack.size());
+    Assert.assertEquals(tokenStack.getFirst(), result);
+    verifyZeroInteractions(variableResolver);
+    verifyZeroInteractions(functionResolver);
+    verifyZeroInteractions(context);
+    verifyZeroInteractions(arithmeticEvaluator);
+    verifyZeroInteractions(comparisonExpressionWithOperatorEvaluator);
+  }
+
+  @Test
+  public void properlyCompareTwoNumbers() throws Exception {
+    StellarParser.ComparisonExpressionWithOperatorContext ctx = mock(StellarParser.ComparisonExpressionWithOperatorContext.class);
+    StellarParser.ComparisonOpContext mockOp = mock(StellarParser.ComparisonOpContext.class);
+    when(ctx.comp_operator()).thenReturn(mockOp);
+    Token result = mock(Token.class);
+    when(comparisonExpressionWithOperatorEvaluator.evaluate(any(Token.class), any(Token.class), any(StellarParser.ComparisonOpContext.class))).thenReturn(result);
+
+    compiler.exitComparisonExpressionWithOperator(ctx);
+    Assert.assertEquals(1, tokenStack.size());
+    StellarCompiler.DeferredFunction func = (StellarCompiler.DeferredFunction) tokenStack.pop().getValue();
+    tokenStack.push(new Token<>(1000, Integer.class));
+    tokenStack.push(new Token<>(1500f, Float.class));
+    func.apply(tokenStack, new StellarCompiler.ExpressionState(context, functionResolver, variableResolver));
+    Assert.assertEquals(1, tokenStack.size());
+    Assert.assertEquals(tokenStack.getFirst(), result);
+    verify(comparisonExpressionWithOperatorEvaluator).evaluate(any(Token.class), any(Token.class), eq(mockOp));
+    verifyZeroInteractions(numberLiteralEvaluator);
+    verifyZeroInteractions(variableResolver);
+    verifyZeroInteractions(functionResolver);
+    verifyZeroInteractions(context);
+    verifyZeroInteractions(arithmeticEvaluator);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
index 681bb3d..e1b455b 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
@@ -69,14 +69,17 @@ public class StellarTest {
     {
       String query = "bar:variable";
       Assert.assertEquals("bar", run(query, ImmutableMap.of("bar:variable", "bar")));
+      Assert.assertEquals("grok", run(query, ImmutableMap.of("bar:variable", "grok")));
     }
     {
       String query = "JOIN(['foo', bar:variable], '')";
       Assert.assertEquals("foobar", run(query, ImmutableMap.of("bar:variable", "bar")));
+      Assert.assertEquals("foogrok", run(query, ImmutableMap.of("bar:variable", "grok")));
     }
     {
       String query = "MAP_GET('bar', { 'foo' : 1, 'bar' : bar:variable})";
       Assert.assertEquals("bar", run(query, ImmutableMap.of("bar:variable", "bar")));
+      Assert.assertEquals("grok", run(query, ImmutableMap.of("bar:variable", "grok")));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
index 5c73966..6fec2ec 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java
@@ -65,7 +65,7 @@ public class IndexingConfigFunctionsTest {
     Assert.assertEquals(IndexingConfigurations.getBatchSize((Map<String, Object>) config.get("hdfs")), 10);
   }
 
-  @Test(expected=ParseException.class)
+  @Test(expected=IllegalStateException.class)
   public void testSetBatchBad() {
     run("INDEXING_SET_BATCH(config, 'hdfs', 10)"
                              , new HashMap<>()
@@ -81,7 +81,7 @@ public class IndexingConfigFunctionsTest {
     Assert.assertTrue(IndexingConfigurations.isEnabled((Map<String, Object>) config.get("hdfs")));
   }
 
-  @Test(expected=ParseException.class)
+  @Test(expected=IllegalStateException.class)
   public void testSetEnabledBad() {
     run("INDEXING_SET_ENABLED(config, 'hdfs', 10)"
                              , new HashMap<>()
@@ -97,7 +97,7 @@ public class IndexingConfigFunctionsTest {
     Assert.assertEquals("foo", IndexingConfigurations.getIndex((Map<String, Object>)config.get("hdfs"), null));
   }
 
-  @Test(expected= ParseException.class)
+  @Test(expected= IllegalStateException.class)
   public void testSetIndexBad() {
     run("INDEXING_SET_INDEX(config, 'hdfs', NULL)"
             , new HashMap<>()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/29a0c186/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
index d1d13e8..68be5d3 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
@@ -121,7 +121,7 @@ public class ThreatTriageFunctionsTest {
     Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 );
   }
 
-  @Test(expected=ParseException.class)
+  @Test(expected=IllegalStateException.class)
   public void testAddMalformed() {
     Object o = run(
             "THREAT_TRIAGE_ADD(config, { 'rule': SHELL_GET_EXPRESSION('foo'), 'score' : 10 } )"


Mime
View raw message