incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Added a GC watcher class that can take action when the heap is too high after a old gen GC.
Date Thu, 27 Jun 2013 11:10:36 GMT
Updated Branches:
  refs/heads/master 506e4ce36 -> 46836eb2a


Added a GC watcher class that can take action when the heap is too high after a old gen GC.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/46836eb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/46836eb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/46836eb2

Branch: refs/heads/master
Commit: 46836eb2a29035a4960d4abf4af03e40d6f23a2f
Parents: 506e4ce
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jun 27 07:01:20 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jun 27 07:01:20 2013 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/utils/GCWatcher.java   | 234 +++++++++++++++++++
 1 file changed, 234 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46836eb2/blur-core/src/main/java/org/apache/blur/utils/GCWatcher.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/GCWatcher.java b/blur-core/src/main/java/org/apache/blur/utils/GCWatcher.java
new file mode 100644
index 0000000..fbe0aa5
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/GCWatcher.java
@@ -0,0 +1,234 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+
+public class GCWatcher extends TimerTask {
+
+  private static final Log LOG = LogFactory.getLog(GCWatcher.class);
+  private static final String GET_LAST_GC_INFO = "getLastGcInfo";
+  private static final String CONCURRENT_MARK_SWEEP = "ConcurrentMarkSweep";
+  private static final String CMS_OLD_GEN = "CMS Old Gen";
+
+  private final Timer _timer;
+  private final long _period = TimeUnit.MILLISECONDS.toMillis(25);
+  private GarbageCollectorMXBean _bean;
+  private final double _ratio;
+  private Method _method;
+  private GcInfo _gcInfo;
+  private long _lastIndex;
+  private List<Action> _actions = new ArrayList<Action>();
+  private static GCWatcher _instance;
+
+  public interface Action {
+    void takeAction() throws Exception;
+  }
+
+  private GCWatcher(double ratio) {
+    List<GarbageCollectorMXBean> garbageCollectorMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
+    for (GarbageCollectorMXBean bean : garbageCollectorMXBeans) {
+      String name = bean.getName();
+      if (name.equals(CONCURRENT_MARK_SWEEP)) {
+        _bean = bean;
+        try {
+          _method = _bean.getClass().getDeclaredMethod(GET_LAST_GC_INFO, new Class[] {});
+        } catch (SecurityException e) {
+          throw new RuntimeException(e);
+        } catch (NoSuchMethodException e) {
+          throw new RuntimeException(e);
+        }
+        _method.setAccessible(true);
+      }
+    }
+    _ratio = ratio;
+    if (_bean != null) {
+      _timer = new Timer("gc-watch", true);
+      _timer.schedule(this, _period, _period);
+    } else {
+      _timer = null;
+    }
+  }
+
+  public static void registerAction(Action action) {
+    GCWatcher instance = instance();
+    if (instance != null) {
+      synchronized (instance._actions) {
+        instance._actions.add(action);
+      }
+    }
+  }
+
+  public static synchronized void shutdown() {
+    GCWatcher instance = instance();
+    if (instance != null) {
+      if (instance._timer != null) {
+        instance._timer.purge();
+        instance._timer.cancel();
+      }
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
+      Object gcInfo = getGcInfo(_bean);
+      if (gcInfo != null) {
+        if (_gcInfo == null) {
+          _gcInfo = new GcInfo(gcInfo);
+        } else {
+          _gcInfo.setGcInfo(gcInfo);
+        }
+        if (_lastIndex == _gcInfo.getIndex()) {
+          return;
+        }
+        long startTime = _gcInfo.getStartTime();
+        long endTime = _gcInfo.getEndTime();
+        Map<String, MemoryUsage> usageAfterGc = _gcInfo.getUsageAfterGc();
+        MemoryUsage after = usageAfterGc.get(CMS_OLD_GEN);
+        long used = after.getUsed();
+        long max = after.getMax();
+
+        long totalTime = endTime - startTime;
+        LOG.info("totalTime spent in GC [{0}] collected [{1}]", totalTime, (max - used));
+        long upperLimit = (long) (max * _ratio);
+        if (used > upperLimit) {
+          LOG.error("WARNING!!!! - Heap used [{0}] over limit of [{1}], taking action to
avoid an OOM error.", used,
+              upperLimit);
+          synchronized (_actions) {
+            for (Action action : _actions) {
+              try {
+                action.takeAction();
+              } catch (Exception e) {
+                LOG.error("Unknown error while trying to take action against an OOM [{0}]",
e, action);
+              }
+            }
+          }
+        }
+        _lastIndex = _gcInfo.getIndex();
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  private Object getGcInfo(GarbageCollectorMXBean bean) throws Exception {
+    return _method.invoke(bean, new Object[] {});
+  }
+
+  static class GcInfo {
+
+    private Object _gcInfo;
+    private final Field _startTimeField;
+    private final Field _endTimeField;
+    private final Field _usageAfterGcField;
+    private final Field _usageBeforeGcField;
+    private final Field _indexField;
+
+    GcInfo(Object o) throws SecurityException, NoSuchFieldException {
+      Class<? extends Object> clazz = o.getClass();
+      _startTimeField = setup(clazz.getDeclaredField("startTime"));
+      _endTimeField = setup(clazz.getDeclaredField("endTime"));
+      _usageAfterGcField = setup(clazz.getDeclaredField("usageAfterGc"));
+      _usageBeforeGcField = setup(clazz.getDeclaredField("usageBeforeGc"));
+      _indexField = setup(clazz.getDeclaredField("index"));
+      setGcInfo(o);
+    }
+
+    void setGcInfo(Object o) {
+      _gcInfo = o;
+    }
+
+    private Field setup(Field field) {
+      field.setAccessible(true);
+      return field;
+    }
+
+    long getStartTime() {
+      try {
+        return _startTimeField.getLong(_gcInfo);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    long getEndTime() {
+      try {
+        return _endTimeField.getLong(_gcInfo);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    long getIndex() {
+      try {
+        return _indexField.getLong(_gcInfo);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @SuppressWarnings("unchecked")
+    Map<String, MemoryUsage> getUsageAfterGc() {
+      try {
+        return (Map<String, MemoryUsage>) _usageAfterGcField.get(_gcInfo);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @SuppressWarnings("unchecked")
+    Map<String, MemoryUsage> getUsageBeforeGc() {
+      try {
+        return (Map<String, MemoryUsage>) _usageBeforeGcField.get(_gcInfo);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+  }
+
+  public synchronized static void init(double ratio) {
+    if (_instance == null) {
+      try {
+        _instance = new GCWatcher(ratio);
+      } catch (Exception e) {
+
+      }
+    } else {
+      LOG.warn("GCWatcher has already been initialized");
+    }
+  }
+
+  private static GCWatcher instance() {
+    return _instance;
+  }
+
+}


Mime
View raw message