brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [4/6] brooklyn-server git commit: control over clearing soft references (default disabled)
Date Mon, 26 Sep 2016 10:37:58 GMT
control over clearing soft references (default disabled)

and tidier memory usage reporting


Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/efd337d9
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/efd337d9
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/efd337d9

Branch: refs/heads/master
Commit: efd337d92d340d8f98e516e9f466be3c89586541
Parents: efb0742
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Fri Sep 16 14:27:26 2016 +0100
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Fri Sep 16 16:53:20 2016 +0100

----------------------------------------------------------------------
 .../mgmt/internal/BrooklynGarbageCollector.java | 34 +++++++++++----
 .../util/javalang/MemoryUsageTracker.java       | 45 +++++++++++++++++++-
 .../util/javalang/MemoryUsageTrackerTest.java   | 34 ++++++++++++++-
 3 files changed, 102 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
index c8ef0e6..75e663a 100644
--- a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
+++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
@@ -60,8 +60,8 @@ import org.apache.brooklyn.util.time.Duration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Objects;
 import com.google.common.annotations.Beta;
+import com.google.common.base.Objects;
 import com.google.common.collect.Iterables;
 
 /**
@@ -92,8 +92,17 @@ public class BrooklynGarbageCollector {
             Duration.minutes(1));
     
     public static final ConfigKey<Boolean> DO_SYSTEM_GC = ConfigKeys.newBooleanConfigKey(
-            "brooklyn.gc.doSystemGc", "whether to periodically call System.gc()", false);
-    
+        "brooklyn.gc.doSystemGc", "whether to periodically call System.gc()", false);
+
+    public static final ConfigKey<Double> FORCE_CLEAR_SOFT_REFERENCES_ON_MEMORY_USAGE_LEVEL
= 
+        ConfigKeys.newDoubleConfigKey("brooklyn.gc.clearSoftReferencesOnMemoryUsageLevel",

+            "force clearance of soft references (by generating a deliberate OOME) "
+            + "if memory usage gets higher than this percentage of available memory; "
+            + "Brooklyn will use up to the max, or this percentage, with soft references,"
+            + "so if using any high-memory-usage alerts they should be pegged quite a bit"
+            + "higher than this threshhold "
+            + "(default >1 means never)", 2.0);
+
     /** 
      * should we check for tasks which are submitted by another but backgrounded, i.e. not
a child of that task?
      * default to yes, despite it can be some extra loops, to make sure we GC them promptly.
@@ -142,6 +151,7 @@ public class BrooklynGarbageCollector {
     };
     
     private final BasicExecutionManager executionManager;
+    @SuppressWarnings("unused")  // TODO remove BrooklynStorage altogether?
     private final BrooklynStorage storage;
     private final BrooklynProperties brooklynProperties;
     private final ScheduledExecutorService executor;
@@ -149,7 +159,6 @@ public class BrooklynGarbageCollector {
     private Map<Entity,Task<?>> unmanagedEntitiesNeedingGc = new LinkedHashMap<Entity,
Task<?>>();
     
     private Duration gcPeriod;
-    private final boolean doSystemGc;
     private volatile boolean running = true;
     
     public BrooklynGarbageCollector(BrooklynProperties brooklynProperties, BasicExecutionManager
executionManager, BrooklynStorage storage) {
@@ -157,7 +166,6 @@ public class BrooklynGarbageCollector {
         this.storage = storage;
         this.brooklynProperties = brooklynProperties;
 
-        doSystemGc = brooklynProperties.getConfig(DO_SYSTEM_GC);
         
         executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
                 @Override public Thread newThread(Runnable r) {
@@ -196,7 +204,13 @@ public class BrooklynGarbageCollector {
             gcTasks();
             logUsage("brooklyn gc (after)");
             
-            if (doSystemGc) {
+            double memUsage = 1.0 - 1.0*Runtime.getRuntime().freeMemory() / Runtime.getRuntime().totalMemory();
+            if (memUsage > brooklynProperties.getConfig(FORCE_CLEAR_SOFT_REFERENCES_ON_MEMORY_USAGE_LEVEL))
{
+                LOG.info("Forcing brooklyn gc including soft references due to memory usage:
"+getUsageString());
+                MemoryUsageTracker.forceClearSoftReferences();
+                System.gc(); System.gc();
+                LOG.info("Forced thorough brooklyn gc, usage now: "+getUsageString());
+            } else if (brooklynProperties.getConfig(DO_SYSTEM_GC)) {
                 // Can be very useful when tracking down OOMEs etc, where a lot of tasks
are executing
                 // Empirically observed that (on OS X jvm at least) calling twice blocks
- logs a significant
                 // amount of memory having been released, as though a full-gc had been run.
But this is highly
@@ -219,13 +233,17 @@ public class BrooklynGarbageCollector {
     public static String makeBasicUsageString() {
         return Strings.makeSizeString(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())+"
/ "+
             Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory" +
-            " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed())
+ " soft); "+
+            // don't mention soft references; they're not very accurate
+            // TODO would be nice to give an indication of soft usage or expiry rate;
+            // a sampling of Maybe instances could be useful for that
+//            " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed())
+ " soft); "+
             Thread.activeCount()+" threads";
     }
     
     public String getUsageString() {
         return makeBasicUsageString()+"; "+
-            "storage: " + storage.getStorageMetrics() + "; " +
+            // ignore storage
+//            "storage: " + storage.getStorageMetrics() + "; " +
             "tasks: " +
             executionManager.getNumActiveTasks()+" active, "+
             executionManager.getNumIncompleteTasks()+" unfinished; "+

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
index e3897cc..7d09efd 100644
--- a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
+++ b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
@@ -20,8 +20,12 @@ package org.apache.brooklyn.util.javalang;
 
 import java.lang.ref.SoftReference;
 import java.lang.ref.WeakReference;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.brooklyn.util.collections.MutableList;
+import org.apache.brooklyn.util.text.Strings;
+
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.RemovalListener;
@@ -44,7 +48,9 @@ public class MemoryUsageTracker {
      * the amount of used memory which is reclaimable by collecting soft references.
      * <p>
      * This is particularly handy for tracking {@link SoftReference}s, because otherwise
you can quickly get to a state
-     * where {@link Runtime#freeMemory()} looks very low.
+     * where {@link Runtime#freeMemory()} <i>looks</i> very low.
+     * <p>
+     * Also consider {@link #forceClearSoftReferences()} to get useful information.
      **/
     public static final MemoryUsageTracker SOFT_REFERENCES = new MemoryUsageTracker();
     
@@ -68,5 +74,40 @@ public class MemoryUsageTracker {
         memoryTrackedReferences.cleanUp();
         return bytesUsed.get();
     }
-    
+
+    /** forces all soft references to be cleared by trying to allocate an enormous chunk
of memory,
+     * returns a description of what was done 
+     * (tune with with {@link #forceClearSoftReferences(long, int)} 
+     * for greater than 200M precision in the output message, if you really care about that)
*/
+    public static String forceClearSoftReferences() {
+        return forceClearSoftReferences(1000*1000, Integer.MAX_VALUE);
+    }
+    /** as {@link #forceClearSoftReferences()} but gives control over headroom and max chunk
size.
+     * it tries to undershoot by headroom as it approaches maximum (and then overshoot)
+     * to minimize the chance we take exactly all the memory and starve another thread;
+     * and it uses the given max chunk size in cases where the caller wants more precision
+     * (the available memory will be fragmented so the smaller the chunks the more it can
+     * fill in, but at the expense of time and actual memory provisioned) */
+    public static String forceClearSoftReferences(long headroom, int maxChunk) {
+        final long HEADROOM = 1000*1000;  
+        long lastAmount = 0;
+        long nextAmount = 0;
+        try {
+            List<byte[]> dd = MutableList.of();
+            while (true) {
+                int size = (int)Math.min(Runtime.getRuntime().freeMemory()-HEADROOM, maxChunk);
+                if (size<HEADROOM) {
+                    // do this to minimize the chance another thread gets an OOME
+                    // due to us leaving just a tiny amount of memory 
+                    size = (int) Math.min(size + 2*HEADROOM, maxChunk);
+                }
+                nextAmount += size;
+                dd.add(new byte[size]);
+                lastAmount = nextAmount;
+            }
+        } catch (OutOfMemoryError e) { /* expected */ }
+        return "allocated " + Strings.makeSizeString((lastAmount+nextAmount)/2) +
+                (lastAmount<nextAmount ? " +- "+Strings.makeSizeString((nextAmount-lastAmount)/2)
: "")
+                +" really free memory in "+Strings.makeSizeString(maxChunk)+" chunks";
+    }
 }

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
----------------------------------------------------------------------
diff --git a/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
b/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
index 80e4e7f..f995e25 100644
--- a/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
+++ b/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
@@ -81,7 +81,39 @@ public class MemoryUsageTrackerTest {
         }
         return size;
     }
-    
+
+    @Test(groups="Integration")
+    public void testSoftUsageAndClearance() {
+        long totalMemory = Runtime.getRuntime().totalMemory();
+        long freeMemory = Runtime.getRuntime().freeMemory();
+        
+        List<Maybe<?>> dump = MutableList.of();
+        Maybe<byte[]> first = Maybe.soft(new byte[1000*1000]);
+        dump.add(first);
+        for (int i=0; i<1000*1000; i++) {
+            totalMemory = Runtime.getRuntime().totalMemory();
+            freeMemory = Runtime.getRuntime().freeMemory();
+            
+            dump.add(Maybe.soft(new byte[1000*1000]));
+            if (first.isAbsent()) break;
+        }
+        int cleared = 0;
+        for (Maybe<?> m: dump) { if (m.isAbsent()) cleared++; }
+        LOG.info("First soft reference cleared after "+dump.size()+" 1M blocks created; "+cleared+"
of them cleared");
+        
+        Assert.assertTrue(1.0*freeMemory/totalMemory < 0.10, 
+            "Should have had less than 10% free memory before clearance, had "+Strings.makeSizeString(freeMemory)+"
/ "+Strings.makeSizeString(totalMemory));
+        
+        LOG.info("Forcing memory eviction: "+
+            MemoryUsageTracker.forceClearSoftReferences(100*1000, 10*1000*1000));
+        
+        System.gc(); System.gc();
+        totalMemory = Runtime.getRuntime().totalMemory();
+        freeMemory = Runtime.getRuntime().freeMemory();
+        Assert.assertTrue(1.0*freeMemory/totalMemory > 0.90, 
+            "Should now have more than 90% free memory, had "+Strings.makeSizeString(freeMemory)+"
/ "+Strings.makeSizeString(totalMemory));
+    }
+        
     private static void assertLessThan(long lhs, long rhs) {
         Assert.assertTrue(lhs<rhs, "Expected "+lhs+" < "+rhs);
     }


Mime
View raw message