metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmiklav...@apache.org
Subject metron git commit: METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251
Date Wed, 07 Nov 2018 01:10:06 GMT
Repository: metron
Updated Branches:
  refs/heads/master b9461e765 -> 85cd21aa0


METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes
apache/metron#1251


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85cd21aa
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85cd21aa
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85cd21aa

Branch: refs/heads/master
Commit: 85cd21aa0f5045184c168248dc2b81c1cfd41ddd
Parents: b9461e7
Author: mmiklavc <michael.miklavcic@gmail.com>
Authored: Tue Nov 6 18:09:56 2018 -0700
Committer: Michael Miklavcic <michael.miklavcic@gmail.com>
Committed: Tue Nov 6 18:09:56 2018 -0700

----------------------------------------------------------------------
 .../ElasticsearchSearchIntegrationTest.java     |   1 -
 .../metron/stellar/dsl/StellarFunction.java     |   9 +-
 .../metron/stellar/dsl/StellarFunctions.java    |   5 +
 .../resolver/BaseFunctionResolver.java          |  44 +++++
 .../functions/resolver/FunctionResolver.java    |  14 +-
 .../stellar/dsl/functions/BasicStellarTest.java |  20 ++-
 .../resolver/BaseFunctionResolverTest.java      | 169 +++++++++++++++++++
 7 files changed, 251 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 1d2d48e..8187468 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -25,7 +25,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.elasticsearch.dao.ElasticsearchDao;

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
index efdd185..4fabfaf 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
@@ -17,10 +17,17 @@
  */
 package org.apache.metron.stellar.dsl;
 
+import java.io.Closeable;
+import java.io.IOException;
 import java.util.List;
 
-public interface StellarFunction {
+public interface StellarFunction extends Closeable {
   Object apply(List<Object> args, Context context) throws ParseException;
   void initialize(Context context);
   boolean isInitialized();
+
+  @Override
+  default void close() throws IOException {
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
index dfec90e..73df82f 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
@@ -18,6 +18,7 @@
 
 package org.apache.metron.stellar.dsl;
 
+import java.io.IOException;
 import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
 import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
 
@@ -30,4 +31,8 @@ public class StellarFunctions {
   public static void initialize(Context context) {
     SingletonFunctionResolver.getInstance().initialize(context);
   }
+
+  public static void close() throws IOException {
+    SingletonFunctionResolver.getInstance().close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
index aeed9d9..38a32d1 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
@@ -23,6 +23,7 @@ import static java.lang.String.format;
 import com.google.common.base.Joiner;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
+import java.io.IOException;
 import java.io.Serializable;
 import java.lang.invoke.MethodHandles;
 import java.util.Arrays;
@@ -58,9 +59,15 @@ public abstract class BaseFunctionResolver implements FunctionResolver,
Serializ
    */
   protected Context context;
 
+  /**
+   * Indicates if closed has been called on this resolver.
+   */
+  private boolean closed;
+
   public BaseFunctionResolver() {
     // memoize provides lazy initialization and thread-safety (the ugly cast is necessary
for serialization)
     functions = Suppliers.memoize((Supplier<Map<String, StellarFunctionInfo>>
& Serializable) this::resolveFunctions);
+    closed = false;
   }
 
   /**
@@ -95,6 +102,43 @@ public abstract class BaseFunctionResolver implements FunctionResolver,
Serializ
   }
 
   /**
+   * Makes an attempt to close all Stellar functions. Calling close multiple times has no
effect.
+   * @throws IOException Catches all exceptions and summarizes them.
+   */
+  @Override
+  public void close() throws IOException {
+    if (!closed) {
+      LOG.info("Calling close() on Stellar functions.");
+      Map<String, Throwable> errors = new HashMap<>();
+      for (StellarFunctionInfo info : getFunctionInfo()) {
+        try {
+          info.getFunction().close();
+        } catch (Throwable t) {
+          errors.put(info.getName(), t);
+        }
+      }
+      if (!errors.isEmpty()) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("Unable to close Stellar functions:");
+        for (Map.Entry<String, Throwable> e : errors.entrySet()) {
+          Throwable throwable = e.getValue();
+          String eText = String
+              .format("Exception - Function: %s; Message: %s; Cause: %s", e.getKey(),
+                  throwable.getMessage(),
+                  throwable.getCause());
+          sb.append(System.lineSeparator());
+          sb.append(eText);
+        }
+        closed = true;
+        throw new IOException(sb.toString());
+      }
+      closed = true;
+    } else {
+      LOG.info("close() already called on Stellar functions - skipping.");
+    }
+  }
+
+  /**
    * Resolves a function by name.
    * @param functionName The name of the function to resolve.
    * @return The executable StellarFunction.

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
index 5acb42c..4047586 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/FunctionResolver.java
@@ -17,16 +17,17 @@
  */
 package org.apache.metron.stellar.dsl.functions.resolver;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.function.Function;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunction;
 import org.apache.metron.stellar.dsl.StellarFunctionInfo;
 
-import java.util.function.Function;
-
 /**
  * Responsible for function resolution in Stellar.
  */
-public interface FunctionResolver extends Function<String, StellarFunction> {
+public interface FunctionResolver extends Function<String, StellarFunction>, Closeable
{
 
   /**
    * Provides metadata about each Stellar function that is resolvable.
@@ -43,4 +44,11 @@ public interface FunctionResolver extends Function<String, StellarFunction>
{
    * @param context Context used to initialize.
    */
   void initialize(Context context);
+
+  /**
+   * Perform any cleanup necessary for the loaded Stellar functions.
+   */
+  @Override
+  default void close() throws IOException {}
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index dec05a8..79f97bc 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -18,9 +18,20 @@
 
 package org.apache.metron.stellar.dsl.functions;
 
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
+
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.stellar.common.StellarProcessor;
 import org.apache.metron.stellar.dsl.Context;
@@ -37,12 +48,6 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import java.util.*;
-
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
-
 @SuppressWarnings("ALL")
 public class BasicStellarTest {
 
@@ -70,6 +75,7 @@ public class BasicStellarTest {
     public boolean isInitialized() {
       return true;
     }
+
   }
 
   @Stellar(
@@ -96,6 +102,7 @@ public class BasicStellarTest {
     public boolean isInitialized() {
       return true;
     }
+
   }
 
   @Test
@@ -1000,4 +1007,5 @@ public class BasicStellarTest {
     checkFalsey("{}");
     checkFalsey("LIST_ADD([])");
   }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
new file mode 100644
index 0000000..47cbda3
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolverTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.dsl.functions.resolver;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class BaseFunctionResolverTest {
+
+  public static class TestResolver extends BaseFunctionResolver {
+
+    Set<Class<? extends StellarFunction>> classesToResolve = new HashSet<>();
+
+    @Override
+    public Set<Class<? extends StellarFunction>> resolvables() {
+      return classesToResolve;
+    }
+
+    /**
+     * Will attempt to resolve any Stellar functions defined within the specified class.
+     * @param clazz The class which may contain a Stellar function.
+     */
+    public TestResolver withClass(Class<? extends StellarFunction> clazz) {
+      this.classesToResolve.add(clazz);
+      return this;
+    }
+  }
+
+  /**
+   * Often imitated, never duplicated.
+   */
+  @Stellar(namespace = "namespace", name = "afunction", description = "description", returns
= "returns", params = {
+      "param1"})
+  private static class IAmAFunction extends BaseStellarFunction {
+
+    public static int closeCallCount;
+    public static boolean throwException = false; // init here bc of reflection in resolver.
+
+    public IAmAFunction() {
+      closeCallCount = 0;
+    }
+
+    @Override
+    public Object apply(List<Object> args) {
+      return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+      closeCallCount++;
+      if (throwException) {
+        Throwable cause = new Throwable("Some nasty nasty cause.");
+        throw new IOException("Bad things happened", cause);
+      }
+    }
+  }
+
+  /**
+   * Scratch that. I was wrong.
+   */
+  @Stellar(namespace = "namespace", name = "anotherfunction", description = "description",
returns = "returns", params = {
+      "param1"})
+  private static class IAmAnotherFunction extends BaseStellarFunction {
+
+    public static int closeCallCount;
+    public static boolean throwException = false; // init here bc of reflection in resolver.
+
+    public IAmAnotherFunction() {
+      closeCallCount = 0;
+    }
+
+    @Override
+    public Object apply(List<Object> args) {
+      return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+      closeCallCount++;
+      if (throwException) {
+        throw new NullPointerException("A most annoying exception.");
+      }
+    }
+  }
+
+  private TestResolver resolver;
+
+  @Before
+  public void setup() {
+    resolver = new TestResolver();
+    IAmAFunction.throwException = false;
+    IAmAnotherFunction.throwException = false;
+  }
+
+  @Test
+  public void close_calls_all_loaded_function_close_methods() throws IOException {
+    resolver.withClass(IAmAFunction.class);
+    resolver.withClass(IAmAnotherFunction.class);
+    resolver.close();
+    assertThat(IAmAFunction.closeCallCount, equalTo(1));
+    assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+  }
+
+  @Rule
+  public final ExpectedException exception = ExpectedException.none();
+
+  @Test
+  public void close_collects_all_exceptions_thrown_on_loaded_function_close_methods()
+      throws IOException {
+    IAmAFunction.throwException = true;
+    IAmAnotherFunction.throwException = true;
+    resolver.withClass(IAmAFunction.class);
+    resolver.withClass(IAmAnotherFunction.class);
+    exception.expect(IOException.class);
+    resolver.close();
+  }
+
+  @Test
+  public void close_only_throws_exceptions_on_first_invocation()
+      throws IOException {
+    IAmAFunction.throwException = true;
+    IAmAnotherFunction.throwException = true;
+    resolver.withClass(IAmAFunction.class);
+    resolver.withClass(IAmAnotherFunction.class);
+    try {
+      resolver.close();
+      Assert.fail("Should have thrown an exception.");
+    } catch (IOException e) {
+      // intentionally empty
+    }
+    assertThat(IAmAFunction.closeCallCount, equalTo(1));
+    assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+    // should not throw exceptions or call any function's close again.
+    resolver.close();
+    resolver.close();
+    resolver.close();
+    assertThat(IAmAFunction.closeCallCount, equalTo(1));
+    assertThat(IAmAnotherFunction.closeCallCount, equalTo(1));
+  }
+}


Mime
View raw message