lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [1/2] lucenenet git commit: Revert "Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles variable and all references to them" to prevent widespread 'file in use by another process' error.
Date Wed, 13 Sep 2017 22:01:47 GMT
Repository: lucenenet
Updated Branches:
  refs/heads/master 03bc1cbff -> 8eada7984


Revert "Lucene.Net.Store.FSDirectory: Removed Fsync() method and m_staleFiles variable and
all references to them" to prevent widespread 'file in use by another process' error.

This reverts commit bc295b0e79c153d9dfa7e9f99a144db135a014f5.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/681430e4
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/681430e4
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/681430e4

Branch: refs/heads/master
Commit: 681430e44ce316e18b788598ed002f6f14dd3822
Parents: 03bc1cb
Author: Shad Storhaug <shad@shadstorhaug.com>
Authored: Thu Sep 14 04:12:33 2017 +0700
Committer: Shad Storhaug <shad@shadstorhaug.com>
Committed: Thu Sep 14 04:12:33 2017 +0700

----------------------------------------------------------------------
 src/Lucene.Net/Store/FSDirectory.cs |  55 +++++------
 src/Lucene.Net/Util/IOUtils.cs      | 153 +++++++++++++++----------------
 2 files changed, 96 insertions(+), 112 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Store/FSDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs
index 8c65d9c..5487b96 100644
--- a/src/Lucene.Net/Store/FSDirectory.cs
+++ b/src/Lucene.Net/Store/FSDirectory.cs
@@ -92,10 +92,7 @@ namespace Lucene.Net.Store
         public const int DEFAULT_READ_CHUNK_SIZE = 8192;
 
         protected readonly DirectoryInfo m_directory; // The underlying filesystem directory
-
-        // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true)
writes everything to disk before
-        // our FileStream is disposed.
-        //protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>();
// Files written, but not yet sync'ed
+        protected readonly ISet<string> m_staleFiles = new ConcurrentHashSet<string>();
// Files written, but not yet sync'ed
 #pragma warning disable 612, 618
         private int chunkSize = DEFAULT_READ_CHUNK_SIZE;
 #pragma warning restore 612, 618
@@ -313,9 +310,7 @@ namespace Lucene.Net.Store
             {
                 throw new IOException("Cannot delete " + file, e);
             }
-            // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true)
writes everything to disk before
-            // our FileStream is disposed.
-            //m_staleFiles.Remove(name);
+            m_staleFiles.Remove(name);
         }
 
         /// <summary>
@@ -358,35 +353,30 @@ namespace Lucene.Net.Store
 
         protected virtual void OnIndexOutputClosed(FSIndexOutput io)
         {
-            // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true)
writes everything to disk before
-            // our FileStream is disposed.
-            //m_staleFiles.Add(io.name);
+            m_staleFiles.Add(io.name);
         }
 
         public override void Sync(ICollection<string> names)
         {
             EnsureOpen();
+            ISet<string> toSync = new HashSet<string>(names);
+            toSync.IntersectWith(m_staleFiles);
 
-            // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true)
writes everything to disk before
-            // our FileStream is disposed. Therefore, there is nothing else to do in this
method.
-            //ISet<string> toSync = new HashSet<string>(names);
-            //toSync.IntersectWith(m_staleFiles);
-
-            //// LUCENENET specific: Fsync breaks concurrency here.
-            //// Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
-            ////foreach (var name in toSync)
-            ////{
-            ////    Fsync(name);
-            ////}
-
-            //// fsync the directory itsself, but only if there was any file fsynced before
-            //// (otherwise it can happen that the directory does not yet exist)!
-            //if (toSync.Count > 0)
+            // LUCENENET specific: Fsync breaks concurrency here.
+            // Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2
+            //foreach (var name in toSync)
             //{
-            //    IOUtils.Fsync(m_directory.FullName, true);
+            //    Fsync(name);
             //}
 
-            //m_staleFiles.ExceptWith(toSync);
+            // fsync the directory itsself, but only if there was any file fsynced before
+            // (otherwise it can happen that the directory does not yet exist)!
+            if (toSync.Count > 0)
+            {
+                IOUtils.Fsync(m_directory.FullName, true);
+            }
+
+            m_staleFiles.ExceptWith(toSync);
         }
 
         public override string GetLockID()
@@ -545,12 +535,9 @@ namespace Lucene.Net.Store
             // LUCENENET NOTE: FileStream doesn't have a way to set length
         }
 
-        // LUCENENET specific: Fsync is pointless in .NET, since we are 
-        // calling FileStream.Flush(true) before the stream is disposed
-        // which means we never need it at the point in Java where it is called.
-        //protected virtual void Fsync(string name)
-        //{
-        //    IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);           

-        //}
+        protected virtual void Fsync(string name)
+        {
+            IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false);            
+        }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/681430e4/src/Lucene.Net/Util/IOUtils.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs
index 27daafa..5907b60 100644
--- a/src/Lucene.Net/Util/IOUtils.cs
+++ b/src/Lucene.Net/Util/IOUtils.cs
@@ -522,83 +522,80 @@ namespace Lucene.Net.Util
             }
         }
 
-        // LUCENENET specific: Fsync is pointless in .NET, since we are 
-        // calling FileStream.Flush(true) before the stream is disposed
-        // which means we never need it at the point in Java where it is called.
-        //        /// <summary>
-        //        /// Ensure that any writes to the given file is written to the storage
device that contains it. </summary>
-        //        /// <param name="fileToSync"> The file to fsync </param>
-        //        /// <param name="isDir"> If <c>true</c>, the given file
is a directory (we open for read and ignore <see cref="IOException"/>s,
-        //        ///  because not all file systems and operating systems allow to fsync
on a directory) </param>
-        //        public static void Fsync(string fileToSync, bool isDir)
-        //        {
-        //            // Fsync does not appear to function properly for Windows and Linux
platforms. In Lucene version
-        //            // they catch this in IOException branch and return if the call is
for the directory. 
-        //            // In Lucene.Net the exception is UnauthorizedAccessException and is
not handled by
-        //            // IOException block. No need to even attempt to fsync, just return
if the call is for directory
-        //            if (isDir)
-        //            {
-        //                return;
-        //            }
-
-        //            var retryCount = 1;
-        //            while (true)
-        //            {
-        //                FileStream file = null;
-        //                bool success = false;
-        //                try
-        //                {
-        //                    // If the file is a directory we have to open read-only, for
regular files we must open r/w for the fsync to have an effect.
-        //                    // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
-        //                    file = new FileStream(fileToSync,
-        //                        FileMode.Open, // We shouldn't create a file when syncing.
-        //                        // Java version uses FileChannel which doesn't create the
file if it doesn't already exist, 
-        //                        // so there should be no reason for attempting to create
it in Lucene.Net.
-        //                        FileAccess.Write,
-        //                        FileShare.ReadWrite);
-        //                    //FileSupport.Sync(file);
-        //                    file.Flush(true);
-        //                    success = true;
-        //                }
-        //#pragma warning disable 168
-        //                catch (IOException e)
-        //#pragma warning restore 168
-        //                {
-        //                    if (retryCount == 5)
-        //                    {
-        //                        throw;
-        //                    }
-        //#if !NETSTANDARD
-        //                    try
-        //                    {
-        //#endif
-        //                        // Pause 5 msec
-        //                        Thread.Sleep(5);
-        //#if !NETSTANDARD
-        //                    }
-        //                    catch (ThreadInterruptedException ie)
-        //                    {
-        //                        var ex = new ThreadInterruptedException(ie.ToString(),
ie);
-        //                        //ex.AddSuppressed(exc);
-        //                        throw ex;
-        //                    }
-        //#endif
-        //                }
-        //                finally
-        //                {
-        //                    if (file != null)
-        //                    {
-        //                        file.Dispose();
-        //                    }
-        //                }
-
-        //                if (success)
-        //                {
-        //                    return;
-        //                }
-
-        //                retryCount++;
-        //            }
-        //        }
+        /// <summary>
+        /// Ensure that any writes to the given file is written to the storage device that
contains it. </summary>
+        /// <param name="fileToSync"> The file to fsync </param>
+        /// <param name="isDir"> If <c>true</c>, the given file is a directory
(we open for read and ignore <see cref="IOException"/>s,
+        ///  because not all file systems and operating systems allow to fsync on a directory)
</param>
+        public static void Fsync(string fileToSync, bool isDir)
+        {
+            // Fsync does not appear to function properly for Windows and Linux platforms.
In Lucene version
+            // they catch this in IOException branch and return if the call is for the directory.

+            // In Lucene.Net the exception is UnauthorizedAccessException and is not handled
by
+            // IOException block. No need to even attempt to fsync, just return if the call
is for directory
+            if (isDir)
+            {
+                return;
+            }
+
+            var retryCount = 1;
+            while (true)
+            {
+                FileStream file = null;
+                bool success = false;
+                try
+                {
+                    // If the file is a directory we have to open read-only, for regular
files we must open r/w for the fsync to have an effect.
+                    // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
+                    file = new FileStream(fileToSync,
+                        FileMode.Open, // We shouldn't create a file when syncing.
+                        // Java version uses FileChannel which doesn't create the file if
it doesn't already exist, 
+                        // so there should be no reason for attempting to create it in Lucene.Net.
+                        FileAccess.Write,
+                        FileShare.ReadWrite);
+                    //FileSupport.Sync(file);
+                    file.Flush(true);
+                    success = true;
+                }
+#pragma warning disable 168
+                catch (IOException e)
+#pragma warning restore 168
+                {
+                    if (retryCount == 5)
+                    {
+                        throw;
+                    }
+#if !NETSTANDARD
+                    try
+                    {
+#endif
+                        // Pause 5 msec
+                        Thread.Sleep(5);
+#if !NETSTANDARD
+                    }
+                    catch (ThreadInterruptedException ie)
+                    {
+                        var ex = new ThreadInterruptedException(ie.ToString(), ie);
+                        //ex.AddSuppressed(exc);
+                        throw ex;
+                    }
+#endif
+                }
+                finally
+                {
+                    if (file != null)
+                    {
+                        file.Dispose();
+                    }
+                }
+
+                if (success)
+                {
+                    return;
+                }
+
+                retryCount++;
+            }
+        }
     }
 }
\ No newline at end of file


Mime
View raw message