metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [40/44] metron git commit: METRON-964 Add logging in JoinBolt for unexpected cache evictions (merrimanr) closes apache/metron#595
Date Thu, 01 Jun 2017 21:41:59 GMT
METRON-964 Add logging in JoinBolt for unexpected cache evictions (merrimanr) closes apache/metron#595


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f127c6dc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f127c6dc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f127c6dc

Branch: refs/heads/Metron_0.4.0
Commit: f127c6dcfd8c2a2d19314b98eb3840da90d2ab44
Parents: 61105c7
Author: merrimanr <merrimanr@gmail.com>
Authored: Wed May 31 14:54:14 2017 -0500
Committer: merrimanr <merrimanr@apache.org>
Committed: Wed May 31 14:54:14 2017 -0500

----------------------------------------------------------------------
 .../apache/metron/enrichment/bolt/JoinBolt.java | 24 +++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/f127c6dc/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 3bbb3f5..a8e793d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -21,6 +21,9 @@ import com.google.common.base.Joiner;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.Sets;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
@@ -90,11 +93,30 @@ public abstract class JoinBolt<V> extends ConfiguredEnrichmentBolt
{
       }
     };
     cache = CacheBuilder.newBuilder().maximumSize(maxCacheSize)
-            .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES)
+            .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES).removalListener(new JoinRemoveListener())
             .build(loader);
     prepare(map, topologyContext);
   }
 
+  class JoinRemoveListener implements RemovalListener<String, Map<String, V>>
{
+
+    @Override
+    public void onRemoval(RemovalNotification<String, Map<String, V>> removalNotification)
{
+      if (removalNotification.getCause() == RemovalCause.SIZE) {
+        String errorMessage = "Join cache reached max size limit. Increase the maxCacheSize
setting or add more tasks to enrichment/threatintel join bolt.";
+        Exception exception = new Exception(errorMessage);
+        LOG.error(errorMessage, exception);
+        collector.reportError(exception);
+      }
+      if (removalNotification.getCause() == RemovalCause.EXPIRED) {
+        String errorMessage = "Message was in the join cache too long which may be caused
by slow enrichments/threatintels.  Increase the maxTimeRetain setting.";
+        Exception exception = new Exception(errorMessage);
+        LOG.error(errorMessage, exception);
+        collector.reportError(exception);
+      }
+    }
+  }
+
   @SuppressWarnings("unchecked")
   @Override
   public void execute(Tuple tuple) {


Mime
View raw message