metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject incubator-metron git commit: METRON-404 Stellar Compiler Hides Function Initialization Errors (nickwallen) closes apache/incubator-metron#240
Date Tue, 06 Sep 2016 22:37:26 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 8b623b87f -> e3ce452a0


METRON-404 Stellar Compiler Hides Function Initialization Errors (nickwallen) closes apache/incubator-metron#240


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e3ce452a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e3ce452a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e3ce452a

Branch: refs/heads/master
Commit: e3ce452a067fda4941645b152106974d311907a3
Parents: 8b623b8
Author: nickwallen <nick@nickallen.org>
Authored: Tue Sep 6 18:36:59 2016 -0400
Committer: Nick Allen <nick@nickallen.org>
Committed: Tue Sep 6 18:36:59 2016 -0400

----------------------------------------------------------------------
 .../metron/common/stellar/StellarCompiler.java  | 270 ++++++++++---------
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |   3 -
 2 files changed, 142 insertions(+), 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e3ce452a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
index 9239159..d6c60d5 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,24 +20,36 @@ package org.apache.metron.common.stellar;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
-import org.apache.metron.common.dsl.*;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.FunctionMarker;
+import org.apache.metron.common.dsl.FunctionResolver;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.common.dsl.Token;
+import org.apache.metron.common.dsl.VariableResolver;
 import org.apache.metron.common.stellar.generated.StellarBaseListener;
 import org.apache.metron.common.stellar.generated.StellarParser;
 import org.apache.metron.common.utils.ConversionUtils;
 
-import java.util.*;
-import java.util.function.Function;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+import static java.lang.String.format;
 
 public class StellarCompiler extends StellarBaseListener {
+
   private Context context = null;
   private Stack<Token> tokenStack = new Stack<>();
   private FunctionResolver functionResolver;
   private VariableResolver variableResolver;
-  public StellarCompiler( VariableResolver variableResolver
-                        , FunctionResolver functionResolver
-                        , Context context
-                        )
-  {
+
+  public StellarCompiler(VariableResolver variableResolver, FunctionResolver functionResolver,
Context context) {
     this.variableResolver = variableResolver;
     this.functionResolver = functionResolver;
     this.context = context;
@@ -52,35 +64,30 @@ public class StellarCompiler extends StellarBaseListener {
     Object key = null;
 
     Set<Object> set = null;
-    if(left.getValue() instanceof Collection) {
+    if (left.getValue() instanceof Collection) {
       set = new HashSet<>((List<Object>) left.getValue());
-    }
-    else if(left.getValue() != null) {
+    } else if (left.getValue() != null) {
       set = ImmutableSet.of(left.getValue());
-    }
-    else {
+    } else {
       set = new HashSet<>();
     }
 
-
     key = right.getValue();
-    if(key == null || set.isEmpty()) {
+    if (key == null || set.isEmpty()) {
       return false;
     }
     return set.contains(key);
   }
 
   private Double getDouble(Token<?> token) {
-    Number n = (Number)token.getValue();
-    if(n == null) {
+    Number n = (Number) token.getValue();
+    if (n == null) {
       return 0d;
-    }
-    else {
+    } else {
       return n.doubleValue();
     }
   }
 
-
   @Override
   public void exitNullConst(StellarParser.NullConstContext ctx) {
     tokenStack.push(new Token<>(null, Object.class));
@@ -93,10 +100,8 @@ public class StellarCompiler extends StellarBaseListener {
     Double r = getDouble(right);
     Double l = getDouble(left);
     tokenStack.push(new Token<>(l + r, Double.class));
-
   }
 
-
   @Override
   public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) {
     Token<?> right = popStack();
@@ -106,8 +111,6 @@ public class StellarCompiler extends StellarBaseListener {
     tokenStack.push(new Token<>(l - r, Double.class));
   }
 
-
-
   @Override
   public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) {
     Token<?> right = popStack();
@@ -130,22 +133,19 @@ public class StellarCompiler extends StellarBaseListener {
     Token<?> elseExpr = popStack();
     Token<?> thenExpr = popStack();
     Token<?> ifExpr = popStack();
-    boolean b = ((Token<Boolean>)ifExpr).getValue();
-    if(b) {
+    boolean b = ((Token<Boolean>) ifExpr).getValue();
+    if (b) {
       tokenStack.push(thenExpr);
-    }
-    else {
+    } else {
       tokenStack.push(elseExpr);
     }
   }
 
-
   @Override
   public void exitTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx) {
     handleConditional();
   }
 
-
   @Override
   public void exitTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx) {
     handleConditional();
@@ -158,7 +158,6 @@ public class StellarCompiler extends StellarBaseListener {
     tokenStack.push(new Token<>(handleIn(left, right), Boolean.class));
   }
 
-
   @Override
   public void exitNInExpression(StellarParser.NInExpressionContext ctx) {
     Token<?> left = popStack();
@@ -210,7 +209,7 @@ public class StellarCompiler extends StellarBaseListener {
   @Override
   public void exitLogicalConst(StellarParser.LogicalConstContext ctx) {
     Boolean b = null;
-    switch(ctx.getText().toUpperCase()) {
+    switch (ctx.getText().toUpperCase()) {
       case "TRUE":
         b = true;
         break;
@@ -223,45 +222,74 @@ public class StellarCompiler extends StellarBaseListener {
     tokenStack.push(new Token<>(b, Boolean.class));
   }
 
-  private boolean booleanOp(Token<?> left, Token<?> right, BooleanOp op, String
opName)
-  {
+  private boolean booleanOp(Token<?> left, Token<?> right, BooleanOp op, String
opName) {
     Boolean l = ConversionUtils.convert(left.getValue(), Boolean.class);
     Boolean r = ConversionUtils.convert(right.getValue(), Boolean.class);
-    if(l == null || r == null) {
-      throw new ParseException("Unable to operate on " + left.getValue()  + " " + opName
+ " " + right.getValue() + ", null value");
+    if (l == null || r == null) {
+      throw new ParseException("Unable to operate on " + left.getValue() + " " + opName +
" " + right.getValue() + ", null value");
     }
     return op.op(l, r);
-
   }
 
   @Override
   public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) {
-    String funcName = ctx.getChild(0).getText();
-    StellarFunction func = null;
-    try {
-      func = functionResolver.apply(funcName);
-      if(!func.isInitialized()) {
-        func.initialize(context);
-      }
-    }
-    catch(Exception iae) {
-      throw new ParseException("Unable to find string function " + funcName + ".  Valid functions
are "
-              + Joiner.on(',').join(functionResolver.getFunctions())
-      );
-    }
 
-    Token<?> left = popStack();
-    List<Object> argList = null;
-    if(left.getUnderlyingType().equals(List.class)) {
-      argList = (List<Object>) left.getValue();
+    // resolve and initialize the function
+    String functionName = ctx.getChild(0).getText();
+    StellarFunction function = resolveFunction(functionName);
+    initializeFunction(function, functionName);
+
+    // fetch the args, execute, and push result onto the stack
+    List<Object> args = getFunctionArguments(popStack());
+    Object result = function.apply(args, context);
+    tokenStack.push(new Token<>(result, Object.class));
+  }
+
+  /**
+   * Get function arguments.
+   * @param token The token containing the function arguments.
+   * @return
+   */
+  private List<Object> getFunctionArguments(Token<?> token) {
+    if (token.getUnderlyingType().equals(List.class)) {
+      return (List<Object>) token.getValue();
+
+    } else {
+      throw new ParseException("Unable to process in clause because " + token.getValue()
+ " is not a set");
     }
-    else {
-      throw new ParseException("Unable to process in clause because " + left.getValue() +
" is not a set");
+  }
+
+  /**
+   * Resolves a function by name.
+   * @param funcName
+   * @return
+   */
+  private StellarFunction resolveFunction(String funcName) {
+    try {
+      return functionResolver.apply(funcName);
+
+    } catch (Exception e) {
+      String valid = Joiner.on(',').join(functionResolver.getFunctions());
+      String error = format("Unable to resolve function named '%s'.  Valid functions are
%s", funcName, valid);
+      throw new ParseException(error, e);
     }
-    Object result = func.apply(argList, context);
-    tokenStack.push(new Token<>(result, Object.class));
   }
 
+  /**
+   * Initialize a Stellar function.
+   * @param function The function to initialize.
+   * @param functionName The name of the functions.
+   */
+  private void initializeFunction(StellarFunction function, String functionName) {
+    try {
+      if (!function.isInitialized()) {
+        function.initialize(context);
+      }
+    } catch (Throwable t) {
+      String error = format("Unable to initialize function '%s'", functionName);
+      throw new ParseException(error, t);
+    }
+  }
 
   @Override
   public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) {
@@ -278,12 +306,11 @@ public class StellarCompiler extends StellarBaseListener {
   @Override
   public void exitFunc_args(StellarParser.Func_argsContext ctx) {
     LinkedList<Object> args = new LinkedList<>();
-    while(true) {
+    while (true) {
       Token<?> token = popStack();
-      if(token.getUnderlyingType().equals(FunctionMarker.class)) {
+      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
         break;
-      }
-      else {
+      } else {
         args.addFirst(token.getValue());
       }
     }
@@ -299,93 +326,81 @@ public class StellarCompiler extends StellarBaseListener {
   public void exitMap_entity(StellarParser.Map_entityContext ctx) {
     HashMap<String, Object> args = new HashMap<>();
     Object value = null;
-    for(int i = 0;true;i++) {
+    for (int i = 0; true; i++) {
       Token<?> token = popStack();
-      if(token.getUnderlyingType().equals(FunctionMarker.class)) {
+      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
         break;
-      }
-      else {
-        if(i % 2 == 0) {
+      } else {
+        if (i % 2 == 0) {
           value = token.getValue();
-        }
-        else {
-          args.put(token.getValue() + "" , value);
+        } else {
+          args.put(token.getValue() + "", value);
         }
       }
     }
     tokenStack.push(new Token<>(args, Map.class));
   }
+
   @Override
   public void exitList_entity(StellarParser.List_entityContext ctx) {
     LinkedList<Object> args = new LinkedList<>();
-    while(true) {
+    while (true) {
       Token<?> token = popStack();
-      if(token.getUnderlyingType().equals(FunctionMarker.class)) {
+      if (token.getUnderlyingType().equals(FunctionMarker.class)) {
         break;
-      }
-      else {
+      } else {
         args.addFirst(token.getValue());
       }
     }
     tokenStack.push(new Token<>(args, List.class));
   }
+
   private <T extends Comparable<T>> boolean compare(T l, T r, String op) {
-    if(op.equals("==")) {
-        return l.compareTo(r) == 0;
-      }
-      else if(op.equals("!=")) {
-        return l.compareTo(r) != 0;
-      }
-      else if(op.equals("<")) {
-        return l.compareTo(r) < 0;
-      }
-      else if(op.equals(">")) {
-        return l.compareTo(r) > 0;
-      }
-      else if(op.equals(">=")) {
-        return l.compareTo(r) >= 0;
-      }
-      else {
-        return l.compareTo(r) <= 0;
-      }
+    if (op.equals("==")) {
+      return l.compareTo(r) == 0;
+    } else if (op.equals("!=")) {
+      return l.compareTo(r) != 0;
+    } else if (op.equals("<")) {
+      return l.compareTo(r) < 0;
+    } else if (op.equals(">")) {
+      return l.compareTo(r) > 0;
+    } else if (op.equals(">=")) {
+      return l.compareTo(r) >= 0;
+    } else {
+      return l.compareTo(r) <= 0;
+    }
   }
 
   private boolean compareDouble(Double l, Double r, String op) {
-    if(op.equals("==")) {
-        return Math.abs(l - r) < 1e-6;
-      }
-      else if(op.equals("!=")) {
-        return Math.abs(l - r) >= 1e-6;
-      }
-      else if(op.equals("<")) {
-        return l.compareTo(r) < 0;
-      }
-      else if(op.equals(">")) {
-        return l.compareTo(r) > 0;
-      }
-      else if(op.equals(">=")) {
-        return l.compareTo(r) >= 0;
-      }
-      else {
-        return l.compareTo(r) <= 0;
-      }
+    if (op.equals("==")) {
+      return Math.abs(l - r) < 1e-6;
+    } else if (op.equals("!=")) {
+      return Math.abs(l - r) >= 1e-6;
+    } else if (op.equals("<")) {
+      return l.compareTo(r) < 0;
+    } else if (op.equals(">")) {
+      return l.compareTo(r) > 0;
+    } else if (op.equals(">=")) {
+      return l.compareTo(r) >= 0;
+    } else {
+      return l.compareTo(r) <= 0;
+    }
   }
+
   @Override
   public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext
ctx) {
     String op = ctx.getChild(1).getText();
     Token<?> right = popStack();
     Token<?> left = popStack();
-    if(left.getValue() instanceof Number
-    && right.getValue() instanceof Number
-      ) {
-      Double l = ((Number)left.getValue()).doubleValue();
-      Double r = ((Number)right.getValue()).doubleValue();
+    if (left.getValue() instanceof Number
+            && right.getValue() instanceof Number) {
+      Double l = ((Number) left.getValue()).doubleValue();
+      Double r = ((Number) right.getValue()).doubleValue();
       tokenStack.push(new Token<>(compareDouble(l, r, op), Boolean.class));
 
-    }
-    else {
-      String l = left.getValue() == null?"":left.getValue().toString();
-      String r = right.getValue() == null?"":right.getValue().toString();
+    } else {
+      String l = left.getValue() == null ? "" : left.getValue().toString();
+      String r = right.getValue() == null ? "" : right.getValue().toString();
       tokenStack.push(new Token<>(compare(l, r, op), Boolean.class));
     }
   }
@@ -396,24 +411,23 @@ public class StellarCompiler extends StellarBaseListener {
   }
 
   public Token<?> popStack() {
-    if(tokenStack.empty()) {
+    if (tokenStack.empty()) {
       throw new ParseException("Unable to pop an empty stack");
     }
     return tokenStack.pop();
   }
 
   public Object getResult() throws ParseException {
-    if(tokenStack.empty()) {
+    if (tokenStack.empty()) {
       throw new ParseException("Invalid predicate: Empty stack.");
     }
     Token<?> token = popStack();
-    if(tokenStack.empty()) {
+    if (tokenStack.empty()) {
       return token.getValue();
     }
-    if(tokenStack.empty()) {
+    if (tokenStack.empty()) {
       throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(tokenStack));
-    }
-    else {
+    } else {
       throw new ParseException("Invalid parse, found " + token);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e3ce452a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 6635dd6..7ff8739 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -19,14 +19,12 @@ package org.apache.metron.enrichment.bolt;
 
 import backtype.storm.task.TopologyContext;
 import com.google.common.base.Joiner;
-import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
 import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
 import org.apache.metron.common.dsl.Context;
 import org.apache.metron.common.dsl.FunctionResolver;
 import org.apache.metron.common.dsl.StellarFunctions;
-import org.apache.metron.common.dsl.functions.ConversionFunctions;
 import org.apache.metron.common.utils.ConversionUtils;
 import org.apache.metron.common.utils.MessageUtils;
 import org.apache.metron.threatintel.triage.ThreatTriageProcessor;
@@ -35,7 +33,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {


Mime
View raw message