metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject metron git commit: METRON-1209: Make stellar repl take logging properties, like other CLI apps in metron closes apache/incubator-metron#772
Date Tue, 03 Oct 2017 14:06:21 GMT
Repository: metron
Updated Branches:
  refs/heads/master 0cc0e2ee0 -> 6ffff0299


METRON-1209: Make stellar repl take logging properties, like other CLI apps in metron closes
apache/incubator-metron#772


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6ffff029
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6ffff029
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6ffff029

Branch: refs/heads/master
Commit: 6ffff0299e75664dc36bbd014668a019cd1d00f2
Parents: 0cc0e2e
Author: cstella <cestella@gmail.com>
Authored: Tue Oct 3 10:06:12 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Tue Oct 3 10:06:12 2017 -0400

----------------------------------------------------------------------
 metron-stellar/pom.xml                          |   1 +
 .../stellar-3rd-party-example/README.md         |   6 +++
 .../stellar-3rd-party-example/pom.xml           |  39 ++++++++++++++++
 .../metron/stellar/external/TimeFunction.java   |  44 +++++++++++++++++++
 .../stellar/common/shell/StellarShell.java      |  19 +++++---
 .../resolver/ClasspathFunctionResolver.java     |  10 ++++-
 .../classpath-resources/custom-1.0-SNAPSHOT.jar | Bin 3570 -> 3968 bytes
 .../resolver/ClasspathFunctionResolverTest.java |  15 -------
 8 files changed, 110 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml
index fd62786..075ceb5 100644
--- a/metron-stellar/pom.xml
+++ b/metron-stellar/pom.xml
@@ -41,6 +41,7 @@
         </license>
     </licenses>
     <modules>
+        <module>stellar-3rd-party-example</module>
         <module>stellar-common</module>
     </modules>
     <dependencies>

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-3rd-party-example/README.md b/metron-stellar/stellar-3rd-party-example/README.md
new file mode 100644
index 0000000..14c489e
--- /dev/null
+++ b/metron-stellar/stellar-3rd-party-example/README.md
@@ -0,0 +1,6 @@
+# Introduction
+
+This is just a sample 3rd party stellar function (which is used in our
+unit testing).  If Stellar changes its API (e.g. the annotation changes
+classes), then this will need to be built and the output jar of this (in
+`target`) will need to be placed in `../stellar-common/src/test/classpath-resources/`.

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-3rd-party-example/pom.xml b/metron-stellar/stellar-3rd-party-example/pom.xml
new file mode 100644
index 0000000..119892c
--- /dev/null
+++ b/metron-stellar/stellar-3rd-party-example/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software
+    Foundation (ASF) under one or more contributor license agreements. See the
+    NOTICE file distributed with this work for additional information regarding
+    copyright ownership. The ASF licenses this file to You under the Apache License,
+    Version 2.0 (the "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software distributed
+    under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+    OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+  the specific language governing permissions and limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+      <artifactId>metron-stellar</artifactId>
+      <groupId>org.apache.metron</groupId>
+      <version>0.4.1</version>
+    </parent>
+    <artifactId>stellar-3rd-party-example</artifactId>
+    <packaging>jar</packaging>
+
+    <name>3rd party Functions (just for tests)</name>
+    <properties>
+      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+      <dependency>
+        <groupId>org.apache.metron</groupId>
+        <artifactId>stellar-common</artifactId>
+        <version>${project.parent.version}</version>
+        <scope>provided</scope>
+      </dependency>
+    </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java
b/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java
new file mode 100644
index 0000000..887531e
--- /dev/null
+++ b/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.external;
+
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import java.util.List;
+
+public class TimeFunction {
+  @Stellar( name="NOW",
+            description = "Right now!",
+            params = {},
+            returns="Timestamp"
+          )
+  public static class Now implements StellarFunction {
+    
+    public Object apply(List<Object> list, Context context) throws ParseException {
+      return System.currentTimeMillis();
+    }
+    
+    public void initialize(Context context) { }
+    
+    public boolean isInitialized() {
+      return true;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
index 0002c5a..6f23a5e 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
@@ -24,12 +24,9 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
+import org.apache.commons.cli.*;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.PropertyConfigurator;
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.utils.JSONUtils;
 import org.apache.metron.stellar.dsl.StellarFunctionInfo;
@@ -124,7 +121,12 @@ public class StellarShell extends AeshConsoleCallback implements Completion
{
     options.addOption("na", "no_ansi", false, "Make the input prompt not use ANSI colors.");
     options.addOption("h", "help", false, "Print help");
     options.addOption("p", "properties", true, "File containing Stellar properties");
-
+    {
+      Option o = new Option("l", "log4j", true, "The log4j properties file to load");
+      o.setArgName("FILE");
+      o.setRequired(false);
+      options.addOption(o);
+    }
     CommandLineParser parser = new PosixParser();
     CommandLine commandLine = parser.parse(options, args);
 
@@ -141,7 +143,10 @@ public class StellarShell extends AeshConsoleCallback implements Completion
{
       System.out.println(e.getMessage());
       System.exit(1);
     }
-    
+    //setting up logging if specified
+    if(commandLine.hasOption("l")) {
+      PropertyConfigurator.configure(commandLine.getOptionValue("l"));
+    }
     console = createConsole(commandLine);
     executor = createExecutor(commandLine, console, getStellarProperties(commandLine));
     loadVariables(commandLine, executor);

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
index 41de982..85aa015 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
@@ -234,7 +234,9 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver {
     else {
       cls = new ClassLoader[this.classLoaders.size()];
       for (int i = 0; i < this.classLoaders.size(); ++i) {
-        cls[i] = this.classLoaders.get(i);
+        ClassLoader cl = this.classLoaders.get(i);
+        LOG.debug("Using classloader: "+ cl.getClass().getCanonicalName());
+        cls[i] = cl;
       }
     }
 
@@ -253,9 +255,13 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver {
     Set<Class<? extends StellarFunction>> ret = new HashSet<>();
     for(ClassLoader cl : cls) {
       for(Class<?> c : ClassIndex.getAnnotated(Stellar.class, cl)) {
-        if(StellarFunction.class.isAssignableFrom(c) && filterBuilder.apply(c.getCanonicalName()))
{
+        LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName());
+        boolean isAssignable = StellarFunction.class.isAssignableFrom(c);
+        boolean isFiltered = filterBuilder.apply(c.getCanonicalName());
+        if( isAssignable && isFiltered ) {
           String className = c.getName();
           if(!classes.contains(className)) {
+            LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className);
             ret.add((Class<? extends StellarFunction>) c);
             classes.add(className);
           }

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
b/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
index 7007eda..9ce87e7 100644
Binary files a/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
and b/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar differ

http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java
index 72cd53a..1d37f99 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java
@@ -109,7 +109,6 @@ public class ClasspathFunctionResolverTest {
     Assert.assertEquals(0, actual.size());
   }
 
-  @Ignore  //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
   @Test
   public void testExternalLocal() throws FileSystemException, ClassNotFoundException {
     File jar = new File("src/test/classpath-resources");
@@ -122,18 +121,4 @@ public class ClasspathFunctionResolverTest {
     Assert.assertTrue(functions.contains("NOW"));
   }
 
-
-  @Ignore  //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar
-  @Test
-  public void testExternalHDFS() throws FileSystemException, ClassNotFoundException {
-    /* TODO: this needs to test HDFS, not local filesystem */
-    File jar = new File("src/test/classpath-resources");
-    Assert.assertTrue(jar.exists());
-    Properties config = new Properties();
-    config.put(STELLAR_VFS_PATHS.param(), jar.toURI() + "/.*.jar");
-
-    ClasspathFunctionResolver resolver = create(config);
-    HashSet<String> functions = new HashSet<>(Lists.newArrayList(resolver.getFunctions()));
-    Assert.assertTrue(functions.contains("NOW"));
-  }
 }


Mime
View raw message