lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject [Lucene.Net] svn commit: r1162583 - in /incubator/lucene.net/trunk: src/core/SupportClass.cs src/core/Util/CloseableThreadLocal-old.cs src/core/Util/CloseableThreadLocal.cs test/core/TestSupportClass.cs
Date Sun, 28 Aug 2011 20:23:02 GMT
Author: digy
Date: Sun Aug 28 20:23:01 2011
New Revision: 1162583

URL: http://svn.apache.org/viewvc?rev=1162583&view=rev
Log:
[LUCENENET-358] MemoryLeaks
See
http://groups.google.com/group/ravendb/browse_thread/thread/9c6340987407aece/1d86d6a61175233c
and
http://mail-archives.apache.org/mod_mbox/lucene-lucene-net-user/201108.mbox/browser

Added:
    incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal-old.cs
Modified:
    incubator/lucene.net/trunk/src/core/SupportClass.cs
    incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal.cs
    incubator/lucene.net/trunk/test/core/TestSupportClass.cs

Modified: incubator/lucene.net/trunk/src/core/SupportClass.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/SupportClass.cs?rev=1162583&r1=1162582&r2=1162583&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/SupportClass.cs (original)
+++ incubator/lucene.net/trunk/src/core/SupportClass.cs Sun Aug 28 20:23:01 2011
@@ -2309,6 +2309,26 @@ public class SupportClass
         }
     }
 
+    /// <summary>
+    /// For Debuging purposes.
+    /// </summary>
+    public class CloseableThreadLocalProfiler
+    {
+        public static bool _EnableCloseableThreadLocalProfiler = false;
+        public static System.Collections.Generic.List<WeakReference> Instances = new
System.Collections.Generic.List<WeakReference>();
+
+        public static bool EnableCloseableThreadLocalProfiler
+        {
+            get { return _EnableCloseableThreadLocalProfiler; }
+            set
+            {
+                _EnableCloseableThreadLocalProfiler = value;
+                lock (Instances)
+                    Instances.Clear();
+            }
+        }
+    }
+
     public class BuildType
     {
 #if DEBUG

Added: incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal-old.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal-old.cs?rev=1162583&view=auto
==============================================================================
--- incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal-old.cs (added)
+++ incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal-old.cs Sun Aug 28 20:23:01
2011
@@ -0,0 +1,104 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Lucene.Net.Util
+{
+	
+	/// <summary>Java's builtin ThreadLocal has a serious flaw:
+	/// it can take an arbitrarily long amount of time to
+	/// dereference the things you had stored in it, even once the
+	/// ThreadLocal instance itself is no longer referenced.
+	/// This is because there is single, master map stored for
+	/// each thread, which all ThreadLocals share, and that
+	/// master map only periodically purges "stale" entries.
+	/// 
+	/// While not technically a memory leak, because eventually
+	/// the memory will be reclaimed, it can take a long time
+	/// and you can easily hit OutOfMemoryError because from the
+	/// GC's standpoint the stale entries are not reclaimaible.
+	/// 
+	/// This class works around that, by only enrolling
+	/// WeakReference values into the ThreadLocal, and
+	/// separately holding a hard reference to each stored
+	/// value.  When you call {@link #close}, these hard
+	/// references are cleared and then GC is freely able to
+	/// reclaim space by objects stored in it. 
+	/// </summary>
+	
+	public class CloseableThreadLocal
+	{
+		
+		[ThreadStatic]
+        static SupportClass.WeakHashTable slots;
+		
+		public /*protected internal*/ virtual System.Object InitialValue()
+		{
+			return null;
+		}
+
+        public virtual System.Object Get()
+        {
+            object value;
+
+            if (slots == null)
+            {
+                value = InitialValue();
+                if (value != null)
+                    Set(value);
+
+                return value;
+            }
+
+            if (slots.ContainsKey(this))
+            {
+                return slots[this];
+            }
+            else
+            {
+                value = InitialValue();
+                slots[this] = value;
+                return value;
+            }
+        }
+		
+		public virtual void  Set(System.Object object_Renamed)
+		{
+            //+-- For Debuging
+            if (SupportClass.CloseableThreadLocalProfiler.EnableCloseableThreadLocalProfiler
== true)
+            {
+                lock (SupportClass.CloseableThreadLocalProfiler.Instances)
+                {
+                    SupportClass.CloseableThreadLocalProfiler.Instances.Add(new WeakReference(object_Renamed));
+                }
+            }
+            //+--
+
+            if (slots == null)
+                slots = new SupportClass.WeakHashTable();
+
+			slots[this] = object_Renamed;	
+		}
+		
+		public virtual void  Close()
+		{
+            if(slots != null)
+                slots.Remove(this);
+		}
+	}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal.cs?rev=1162583&r1=1162582&r2=1162583&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal.cs (original)
+++ incubator/lucene.net/trunk/src/core/Util/CloseableThreadLocal.cs Sun Aug 28 20:23:01 2011
@@ -16,79 +16,164 @@
  */
 
 using System;
+using System.Collections.Generic;
+using System.Threading;
 
 namespace Lucene.Net.Util
 {
-	
-	/// <summary>Java's builtin ThreadLocal has a serious flaw:
-	/// it can take an arbitrarily long amount of time to
-	/// dereference the things you had stored in it, even once the
-	/// ThreadLocal instance itself is no longer referenced.
-	/// This is because there is single, master map stored for
-	/// each thread, which all ThreadLocals share, and that
-	/// master map only periodically purges "stale" entries.
-	/// 
-	/// While not technically a memory leak, because eventually
-	/// the memory will be reclaimed, it can take a long time
-	/// and you can easily hit OutOfMemoryError because from the
-	/// GC's standpoint the stale entries are not reclaimaible.
-	/// 
-	/// This class works around that, by only enrolling
-	/// WeakReference values into the ThreadLocal, and
-	/// separately holding a hard reference to each stored
-	/// value.  When you call {@link #close}, these hard
-	/// references are cleared and then GC is freely able to
-	/// reclaim space by objects stored in it. 
-	/// </summary>
-	
-	public class CloseableThreadLocal
-	{
-		
-		[ThreadStatic]
-        static SupportClass.WeakHashTable slots;
-		
-		public /*protected internal*/ virtual System.Object InitialValue()
-		{
-			return null;
-		}
 
-        public virtual System.Object Get()
+    /// <summary>Java's builtin ThreadLocal has a serious flaw:
+    /// it can take an arbitrarily long amount of time to
+    /// dereference the things you had stored in it, even once the
+    /// ThreadLocal instance itself is no longer referenced.
+    /// This is because there is single, master map stored for
+    /// each thread, which all ThreadLocals share, and that
+    /// master map only periodically purges "stale" entries.
+    /// 
+    /// While not technically a memory leak, because eventually
+    /// the memory will be reclaimed, it can take a long time
+    /// and you can easily hit OutOfMemoryError because from the
+    /// GC's standpoint the stale entries are not reclaimaible.
+    /// 
+    /// This class works around that, by only enrolling
+    /// WeakReference values into the ThreadLocal, and
+    /// separately holding a hard reference to each stored
+    /// value.  When you call {@link #close}, these hard
+    /// references are cleared and then GC is freely able to
+    /// reclaim space by objects stored in it. 
+    /// </summary>
+    /// 
+
+    public class CloseableThreadLocal
+    {
+        private ThreadLocal<WeakReference> t = new ThreadLocal<WeakReference>();
+
+        private Dictionary<Thread, object> hardRefs = new Dictionary<Thread, object>();
+
+
+        public virtual object InitialValue()
         {
-            object value;
+            return null;
+        }
 
-            if (slots == null)
+        public virtual object Get()
+        {
+            WeakReference weakRef = t.Get();
+            if (weakRef == null)
+            {
+                object iv = InitialValue();
+                if (iv != null)
+                {
+                    Set(iv);
+                    return iv;
+                }
+                else
+                    return null;
+            }
+            else
             {
-                value = InitialValue();
-                if (value != null)
-                    Set(value);
+                return weakRef.Get();
+            }
+        }
 
-                return value;
+        public virtual void Set(object @object)
+        {
+            //+-- For Debuging
+            if (SupportClass.CloseableThreadLocalProfiler.EnableCloseableThreadLocalProfiler
== true)
+            {
+                lock (SupportClass.CloseableThreadLocalProfiler.Instances)
+                {
+                    SupportClass.CloseableThreadLocalProfiler.Instances.Add(new WeakReference(@object));
+                }
             }
+            //+--
 
-            if (slots.ContainsKey(this))
+            t.Set(new WeakReference(@object));
+
+            lock (hardRefs)
             {
-                return slots[this];
+                //hardRefs[Thread.CurrentThread] = @object;
+                hardRefs.Add(Thread.CurrentThread, @object);
+
+                // Purge dead threads
+                foreach (var thread in new List<Thread>(hardRefs.Keys))
+                {
+                    if (!thread.IsAlive)
+                        hardRefs.Remove(thread);
+                }
+
             }
-            else
+        }
+
+        public virtual void Close()
+        {
+            // Clear the hard refs; then, the only remaining refs to
+            // all values we were storing are weak (unless somewhere
+            // else is still using them) and so GC may reclaim them:
+            hardRefs = null;
+            // Take care of the current thread right now; others will be
+            // taken care of via the WeakReferences.
+            if (t != null)
             {
-                value = InitialValue();
-                slots[this] = value;
-                return value;
+                t.Remove();
             }
+            t = null;
+        }
+    }
+
+    internal static class CloseableThreadLocalExtensions
+    {
+        public static void Set<T>(this ThreadLocal<T> t, T val)
+        {
+            t.Value = val;
+        }
+
+        public static T Get<T>(this ThreadLocal<T> t)
+        {
+            return t.Value;
+        }
+
+        public static void Remove<T>(this ThreadLocal<T> t)
+        {
+            t.Dispose();
         }
-		
-		public virtual void  Set(System.Object object_Renamed)
-		{
-            if (slots == null)
-                slots = new SupportClass.WeakHashTable();
-
-			slots[this] = object_Renamed;	
-		}
-		
-		public virtual void  Close()
-		{
-            if(slots != null)
-                slots.Remove(this);
-		}
-	}
+
+        public static object Get(this WeakReference w)
+        {
+            return w.Target;
+        }
+    }
+
+    //// {{DIGY}}
+    //// To compile against Framework 2.0
+    //// Uncomment below class
+    //public class ThreadLocal<T> : IDisposable
+    //{
+    //    [ThreadStatic]
+    //    static SupportClass.WeakHashTable slots;
+
+    //    void Init()
+    //    {
+    //        if (slots == null) slots = new SupportClass.WeakHashTable();
+    //    }
+
+    //    public T Value
+    //    {
+    //        set
+    //        {
+    //            Init();
+    //            slots.Add(this, value);
+    //        }
+    //        get
+    //        {
+    //            Init();
+    //            return (T)slots[this];
+    //        }
+    //    }
+
+    //    public void Dispose()
+    //    {
+    //        slots.Remove(this);
+    //    }
+    //}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/test/core/TestSupportClass.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/test/core/TestSupportClass.cs?rev=1162583&r1=1162582&r2=1162583&view=diff
==============================================================================
--- incubator/lucene.net/trunk/test/core/TestSupportClass.cs (original)
+++ incubator/lucene.net/trunk/test/core/TestSupportClass.cs Sun Aug 28 20:23:01 2011
@@ -210,6 +210,65 @@ namespace Lucene.Net._SupportClass
             int memUsage = GetMemUsageInKB();
             if (memUsage > initialMemUsage * 2) Assert.Fail("Memory Leakage.MemUsage =
" + memUsage);
         }
+
+    }
+
+    [TestFixture]
+    public class TestCloseableThreadLocal
+    {
+        [Test]
+        public void TestMemLeakage()
+        {
+            SupportClass.CloseableThreadLocalProfiler.EnableCloseableThreadLocalProfiler
= true;
+
+            int LoopCount = 100;
+            Analyzer[] analyzers = new Analyzer[LoopCount];
+            RAMDirectory[] dirs = new RAMDirectory[LoopCount];
+            IndexWriter[] indexWriters = new IndexWriter[LoopCount];
+
+            System.Threading.Tasks.Parallel.For(0, LoopCount, (i) =>
+            {
+                analyzers[i] = new Lucene.Net.Analysis.Standard.StandardAnalyzer();
+                dirs[i] = new RAMDirectory();
+                indexWriters[i] = new IndexWriter(dirs[i], analyzers[i], true);
+            });
+
+            System.Threading.Tasks.Parallel.For(0, LoopCount, (i) =>
+            {
+                Document document = new Document();
+                document.Add(new Field("field", "some test", Field.Store.NO, Field.Index.ANALYZED));
+                indexWriters[i].AddDocument(document);
+            });
+
+            System.Threading.Tasks.Parallel.For(0, LoopCount, (i) =>
+            {
+                analyzers[i].Close();
+                indexWriters[i].Close();
+            });
+
+            System.Threading.Tasks.Parallel.For(0, LoopCount, (i) =>
+            {
+                IndexSearcher searcher = new IndexSearcher(dirs[i]);
+                TopDocs d = searcher.Search(new TermQuery(new Term("field", "test")), 10);
+                searcher.Close();
+            });
+
+            System.Threading.Tasks.Parallel.For(0, LoopCount, (i) => dirs[i].Close());
+
+            GC.Collect(GC.MaxGeneration);
+            GC.WaitForPendingFinalizers();
+
+            int aliveObjects = 0;
+            foreach (WeakReference w in SupportClass.CloseableThreadLocalProfiler.Instances)
+            {
+                object o = w.Target;
+                if (o != null) aliveObjects++;
+            }
+
+            SupportClass.CloseableThreadLocalProfiler.EnableCloseableThreadLocalProfiler
= false;
+
+            Assert.AreEqual(0, aliveObjects);
+        }
     }
 
     [TestFixture]



Mime
View raw message