lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [1/4] lucenenet git commit: ENHANCEMENT: Lucene.Net.Store.NativeFSLockFactory: Refactored implementation to take advantage of .NET FileStream.Lock. Implementation provided by Vincent Van Den Berghe.
Date Tue, 25 Jul 2017 08:33:24 GMT
Repository: lucenenet
Updated Branches:
  refs/heads/master c3f60b29f -> db1f605cd


ENHANCEMENT: Lucene.Net.Store.NativeFSLockFactory: Refactored implementation to take advantage
of .NET FileStream.Lock. Implementation provided by Vincent Van Den Berghe.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/7e6b0bca
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/7e6b0bca
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/7e6b0bca

Branch: refs/heads/master
Commit: 7e6b0bcac9e441fb6f17bdb5e324cbbef1bdead3
Parents: c3f60b2
Author: Shad Storhaug <shad@shadstorhaug.com>
Authored: Mon Jul 24 21:06:12 2017 +0700
Committer: Shad Storhaug <shad@shadstorhaug.com>
Committed: Mon Jul 24 21:06:12 2017 +0700

----------------------------------------------------------------------
 src/Lucene.Net/Lucene.Net.csproj            |   2 +-
 src/Lucene.Net/Store/NativeFSLockFactory.cs | 236 +++++++++++++++--------
 src/Lucene.Net/project.json                 |   3 +-
 3 files changed, 162 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e6b0bca/src/Lucene.Net/Lucene.Net.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Lucene.Net.csproj b/src/Lucene.Net/Lucene.Net.csproj
index d2b8e6e..3dc6d30 100644
--- a/src/Lucene.Net/Lucene.Net.csproj
+++ b/src/Lucene.Net/Lucene.Net.csproj
@@ -53,7 +53,7 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <PropertyGroup>
-    <DefineConstants>$(DefineConstants);FEATURE_CLONEABLE;FEATURE_CONCURRENTMERGESCHEDULER;FEATURE_SERIALIZABLE;FEATURE_THREADPOOL_UNSAFEQUEUEWORKITEM</DefineConstants>
+    <DefineConstants>$(DefineConstants);FEATURE_CLONEABLE;FEATURE_CONCURRENTMERGESCHEDULER;FEATURE_SERIALIZABLE;FEATURE_THREADPOOL_UNSAFEQUEUEWORKITEM;FEATURE_FILESTREAM_LOCK</DefineConstants>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="Microsoft.CSharp" />

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e6b0bca/src/Lucene.Net/Store/NativeFSLockFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/Store/NativeFSLockFactory.cs b/src/Lucene.Net/Store/NativeFSLockFactory.cs
index 47b5233..8157e13 100644
--- a/src/Lucene.Net/Store/NativeFSLockFactory.cs
+++ b/src/Lucene.Net/Store/NativeFSLockFactory.cs
@@ -1,26 +1,26 @@
 using Lucene.Net.Util;
 using System;
-using System.Collections.Concurrent;
 using System.IO;
+using System.Collections.Generic;
 
 namespace Lucene.Net.Store
 {
     /*
-     * Licensed to the Apache Software Foundation (ASF) under one or more
-     * contributor license agreements.  See the NOTICE file distributed with
-     * this work for additional information regarding copyright ownership.
-     * The ASF licenses this file to You under the Apache License, Version 2.0
-     * (the "License"); you may not use this file except in compliance with
-     * the License.  You may obtain a copy of the License at
-     *
-     *     http://www.apache.org/licenses/LICENSE-2.0
-     *
-     * Unless required by applicable law or agreed to in writing, software
-     * distributed under the License is distributed on an "AS IS" BASIS,
-     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-     * See the License for the specific language governing permissions and
-     * limitations under the License.
-     */
+    * Licensed to the Apache Software Foundation (ASF) under one or more
+    * contributor license agreements.  See the NOTICE file distributed with
+    * this work for additional information regarding copyright ownership.
+    * The ASF licenses this file to You under the Apache License, Version 2.0
+    * (the "License"); you may not use this file except in compliance with
+    * the License.  You may obtain a copy of the License at
+    *
+    *     http://www.apache.org/licenses/LICENSE-2.0
+    *
+    * Unless required by applicable law or agreed to in writing, software
+    * distributed under the License is distributed on an "AS IS" BASIS,
+    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    * See the License for the specific language governing permissions and
+    * limitations under the License.
+    */
 
     /// <summary>
     /// <para>Implements <see cref="LockFactory"/> using native OS file
@@ -49,7 +49,7 @@ namespace Lucene.Net.Store
     ///
     /// <para>If you suspect that this or any other <see cref="LockFactory"/>
is
     /// not working properly in your environment, you can easily
-    /// test it by using <see cref="VerifyingLockFactory"/>, 
+    /// test it by using <see cref="VerifyingLockFactory"/>,
     /// <see cref="LockVerifyServer"/> and <see cref="LockStressTest"/>.</para>
     /// </summary>
     /// <seealso cref="LockFactory"/>
@@ -89,33 +89,109 @@ namespace Lucene.Net.Store
 
         // LUCENENET: NativeFSLocks in Java are infact singletons; this is how we mimick
that to track instances and make sure
         // IW.Unlock and IW.IsLocked works correctly
-        internal static readonly ConcurrentDictionary<string, Lazy<NativeFSLock>>
_locks = new ConcurrentDictionary<string, Lazy<NativeFSLock>>();
+        internal static readonly Dictionary<string, NativeFSLock> _locks = new Dictionary<string,
NativeFSLock>();
+
+        /// <summary>
+        /// Given a lock name, return the full prefixed path of the actual lock file.
+        /// </summary>
+        /// <param name="lockName"></param>
+        /// <returns></returns>
+        private string GetPathOfLockFile(string lockName)
+        {
+            if (m_lockPrefix != null)
+            {
+                lockName = m_lockPrefix + "-" + lockName;
+            }
+            return Path.Combine(m_lockDir.FullName, lockName);
+        }
 
         public override Lock MakeLock(string lockName)
         {
-            var path = new DirectoryInfo(System.IO.Path.Combine(m_lockDir.FullName, lockName));
-            return _locks.GetOrAdd(path.FullName, s => new Lazy<NativeFSLock>(()
=> new NativeFSLock(this, m_lockDir, s))).Value;
+            var path = GetPathOfLockFile(lockName);
+            NativeFSLock l;
+            lock (_locks)
+                if (!_locks.TryGetValue(path, out l))
+                    _locks.Add(path, l = new NativeFSLock(this, m_lockDir, path));
+            return l;
         }
 
         public override void ClearLock(string lockName)
         {
-            using (var _ = MakeLock(lockName)) { }
+            var path = GetPathOfLockFile(lockName);
+            NativeFSLock l;
+            // this is the reason why we can't use ConcurrentDictionary: we need the removal
and disposal of the lock to be atomic
+            // otherwise it may clash with MakeLock making a lock and ClearLock disposing
of it in another thread.
+            lock (_locks)
+                if (_locks.TryGetValue(path, out l))
+                {
+                    _locks.Remove(path);
+                    l.Dispose();
+                }
         }
     }
 
     internal class NativeFSLock : Lock
     {
+#if FEATURE_FILESTREAM_LOCK
+        private const int ERROR_LOCK_VIOLATION = 0x21;
+#else
+        private const int ERROR_SHARE_VIOLATION = 0x20;
+#endif
+
         private readonly NativeFSLockFactory outerInstance;
 
         private FileStream channel;
-        private readonly DirectoryInfo path;
+        private readonly string path;
         private readonly DirectoryInfo lockDir;
 
-        public NativeFSLock(NativeFSLockFactory outerInstance, DirectoryInfo lockDir, string
lockFileName)
+        public NativeFSLock(NativeFSLockFactory outerInstance, DirectoryInfo lockDir, string
path)
         {
             this.outerInstance = outerInstance;
             this.lockDir = lockDir;
-            path = new DirectoryInfo(System.IO.Path.Combine(lockDir.FullName, lockFileName));
+            this.path = path;
+        }
+
+        /// <summary>
+        /// Return true if the <see cref="IOException"/> is the result of a lock violation
+        /// </summary>
+        /// <param name="e"></param>
+        /// <returns></returns>
+        private static bool IsLockOrShareViolation(IOException e)
+        {
+            var result = e.HResult & 0xFFFF;
+#if FEATURE_FILESTREAM_LOCK
+            return result == ERROR_LOCK_VIOLATION;
+#else
+            return result == ERROR_SHARE_VIOLATION;
+#endif
+        }
+
+        private FileStream GetLockFileStream(FileMode mode)
+        {
+            if (!System.IO.Directory.Exists(lockDir.FullName))
+            {
+                try
+                {
+                    System.IO.Directory.CreateDirectory(lockDir.FullName);
+                }
+                catch (Exception e)
+                {
+                    // note that several processes might have been trying to create the same
directory at the same time.
+                    // if one succeeded, the directory will exist and the exception can be
ignored. In all other cases we should report it.
+                    if (!System.IO.Directory.Exists(lockDir.FullName))
+                        throw new IOException("Cannot create directory: " + lockDir.FullName,
e);
+                }
+            }
+            else if (File.Exists(lockDir.FullName))
+            {
+                throw new IOException("Found regular file where directory expected: " + lockDir.FullName);
+            }
+
+#if FEATURE_FILESTREAM_LOCK
+            return new FileStream(path, mode, FileAccess.Write, FileShare.ReadWrite);
+#else
+            return new FileStream(path, mode, FileAccess.Write, FileShare.None, 1, mode ==
FileMode.Open ? FileOptions.None : FileOptions.DeleteOnClose);
+#endif
         }
 
         public override bool Obtain()
@@ -130,28 +206,48 @@ namespace Lucene.Net.Store
                     return false;
                 }
 
-                if (!System.IO.Directory.Exists(lockDir.FullName))
+#if FEATURE_FILESTREAM_LOCK
+                FileStream stream = null;
+                try
+                {
+                    stream = GetLockFileStream(FileMode.OpenOrCreate);
+                }
+                catch (IOException e)
+                {
+                    FailureReason = e;
+                }
+                // LUCENENET: UnauthorizedAccessException does not derive from IOException
like in java
+                catch (UnauthorizedAccessException e)
+                {
+                    // On Windows, we can get intermittent "Access
+                    // Denied" here.  So, we treat this as failure to
+                    // acquire the lock, but, store the reason in case
+                    // there is in fact a real error case.
+                    FailureReason = e;
+                }
+
+                if (stream != null)
                 {
                     try
                     {
-                        System.IO.Directory.CreateDirectory(lockDir.FullName);
+                        stream.Lock(0, 1);
+                        // only assign the channel if the lock succeeds
+                        channel = stream;
                     }
-                    catch
+                    catch (Exception e)
                     {
-                        throw new IOException("Cannot create directory: " + lockDir.FullName);
+                        FailureReason = e;
+                        IOUtils.DisposeWhileHandlingException(stream);
                     }
                 }
-                else if (File.Exists(lockDir.FullName))
+#else
+                try
                 {
-                    throw new IOException("Found regular file where directory expected: "
+ lockDir.FullName);
+                    channel = GetLockFileStream(FileMode.OpenOrCreate);
                 }
-
-                var success = false;
-                try
+                catch (IOException e) when (IsLockOrShareViolation(e))
                 {
-                    channel = new FileStream(path.FullName, FileMode.Create, FileAccess.Write,
FileShare.None);
-
-                    success = true;
+                    // no failure reason to be recorded, since this is the expected error
if a lock exists
                 }
                 catch (IOException e)
                 {
@@ -166,15 +262,7 @@ namespace Lucene.Net.Store
                     // there is in fact a real error case.
                     FailureReason = e;
                 }
-                finally
-                {
-                    if (!success)
-                    {
-                        IOUtils.DisposeWhileHandlingException(channel);
-                        channel = null;
-                    }
-                }
-
+#endif
                 return channel != null;
             }
         }
@@ -187,28 +275,25 @@ namespace Lucene.Net.Store
                 {
                     if (channel != null)
                     {
-                        IOUtils.DisposeWhileHandlingException(channel);
-                        channel = null;
-
-                        bool tmpBool;
-                        if (File.Exists(path.FullName))
+                        try
                         {
-                            File.Delete(path.FullName);
-                            tmpBool = true;
+                            NativeFSLockFactory._locks.Remove(path);
                         }
-                        else if (System.IO.Directory.Exists(path.FullName))
+                        finally
                         {
-                            System.IO.Directory.Delete(path.FullName);
-                            tmpBool = true;
+                            IOUtils.DisposeWhileHandlingException(channel);
+                            channel = null;
                         }
-                        else
+#if FEATURE_FILESTREAM_LOCK
+                        // try to delete the file if we created it, but it's not an error
if we can't.
+                        try
                         {
-                            tmpBool = false;
+                            File.Delete(path);
                         }
-                        if (!tmpBool)
+                        catch
                         {
-                            throw new LockReleaseFailedException("failed to delete " + path);
                         }
+#endif
                     }
                 }
             }
@@ -218,35 +303,32 @@ namespace Lucene.Net.Store
         {
             lock (this)
             {
-                // The test for is isLocked is not directly possible with native file locks:
-
                 // First a shortcut, if a lock reference in this instance is available
                 if (channel != null)
                 {
                     return true;
                 }
 
-                // Look if lock file is present; if not, there can definitely be no lock!
-                bool tmpBool;
-                if (System.IO.File.Exists(path.FullName))
-                    tmpBool = true;
-                else
-                    tmpBool = System.IO.Directory.Exists(path.FullName);
-                if (!tmpBool)
-                    return false;
-
-                // Try to obtain and release (if was locked) the lock
                 try
                 {
-                    bool obtained = Obtain();
-                    if (obtained)
+                    using (var stream = GetLockFileStream(FileMode.Open))
                     {
-                        Dispose();
+#if FEATURE_FILESTREAM_LOCK
+                        // try to find out if the file is locked by writing a byte. Note
that we need to flush the stream to find out.
+                        stream.WriteByte(0);
+                        stream.Flush();   // this *may* throw an IOException if the file
is locked, but...
+                                          // ... closing the stream is the real test
+#endif
                     }
-                    return !obtained;
+                    return false;
+                }
+                catch (IOException e) when (IsLockOrShareViolation(e))
+                {
+                    return true;
                 }
-                catch (IOException)
+                catch (FileNotFoundException)
                 {
+                    // if the file doesn't exists, there can be no lock active
                     return false;
                 }
             }
@@ -257,4 +339,4 @@ namespace Lucene.Net.Store
             return "NativeFSLock@" + path;
         }
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e6b0bca/src/Lucene.Net/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net/project.json b/src/Lucene.Net/project.json
index 9a38729..b820797 100644
--- a/src/Lucene.Net/project.json
+++ b/src/Lucene.Net/project.json
@@ -46,7 +46,8 @@
           "FEATURE_CLONEABLE",
           "FEATURE_CONCURRENTMERGESCHEDULER",
           "FEATURE_SERIALIZABLE",
-          "FEATURE_THREADPOOL_UNSAFEQUEUEWORKITEM"
+          "FEATURE_THREADPOOL_UNSAFEQUEUEWORKITEM",
+          "FEATURE_FILESTREAM_LOCK"
         ]
       }
     }


Mime
View raw message