lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r593799 - in /lucene/java/trunk: ./ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/ src/java/org/apache/lucene/store/ src/test/org/apache/lucene/index/ src/test/org/apache/lucene/store/
Date Sat, 10 Nov 2007 17:51:03 GMT
Author: mikemccand
Date: Sat Nov 10 09:51:00 2007
New Revision: 593799

URL: http://svn.apache.org/viewvc?rev=593799&view=rev
Log:
LUCENE-1044: add doSync option to FSDirectory.getDirectory, defaulting to true, to sync()
each file descriptor before close()

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
    lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java
    lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Sat Nov 10 09:51:00 2007
@@ -168,8 +168,15 @@
     obtained or released, throwing an exception if an illegal lock
     obtain occurred.  (Patrick Kimber via Mike McCandless)
 
-  6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and
longs.  
-  Added support into SortField for sorting on doubles and longs as well.  (Grant Ingersoll)
+ 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs.
 
+    Added support into SortField for sorting on doubles and longs as well.  (Grant Ingersoll)
+
+ 7. LUCENE-1044: Added optional doSync boolean to
+    FSDirectory.getDirectory(...).  If true (the default) then we will
+    always sync() a file before closing it, which improves the
+    likelihood that the index will remain consistent when the OS or
+    machine crashes, or power to the machine is cut.  (Venkat Rangan
+    via Mike McCandless)
 
 Optimizations
 

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
(original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
Sat Nov 10 09:51:00 2007
@@ -126,7 +126,8 @@
         FileUtils.fullyDelete(indexDir);
       }
       indexDir.mkdirs();
-      directory = FSDirectory.getDirectory(indexDir);
+      final boolean doSync = config.get("fsdirectory.dosync", true);
+      directory = FSDirectory.getDirectory(indexDir, null, doSync);
     } else {
       directory = new RAMDirectory();
     }

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
(original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html
Sat Nov 10 09:51:00 2007
@@ -479,6 +479,12 @@
     </li>
 
     <li>
+    <font color="#FF0066">fsdirectory.dosync</font> - true/false
+    If "true", the FSDirectory will call sync() before close() on each
+    file.
+    </li>
+
+    <li>
     <b>Index work parameters</b>:
     Multi int/boolean values would be iterated with calls to NewRound.
     There would be also added as columns in the reports, first string in the
@@ -543,6 +549,7 @@
     </li><li>merge.factor
     </li><li>max.buffered
     </li><li>directory
+    </li><li>fsdirectory.dosync
     </li><li>ram.flush.mb
     </li><li>autocommit
     </li></ul>
@@ -676,4 +683,4 @@
 </DIV>
 <DIV>&nbsp;</DIV>
 </BODY>
-</HTML>
\ No newline at end of file
+</HTML>

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java Sat Nov 10 09:51:00
2007
@@ -62,6 +62,14 @@
 
   private static boolean disableLocks = false;
 
+  private static boolean DEFAULT_DO_SYNC = true;
+
+  // True if we should call sync() before closing a file.
+  // This improves chances that index will still be
+  // consistent if the machine or OS abruptly crashes.  See
+  // LUCENE-1044.
+  private boolean doSync = DEFAULT_DO_SYNC;
+
   // TODO: should this move up to the Directory base class?  Also: should we
   // make a per-instance (in addition to the static "default") version?
 
@@ -136,17 +144,34 @@
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path)
       throws IOException {
-    return getDirectory(new File(path), null);
+    return getDirectory(new File(path), null, DEFAULT_DO_SYNC);
   }
 
   /** Returns the directory instance for the named location.
    * @param path the path to the directory.
    * @param lockFactory instance of {@link LockFactory} providing the
-   *        locking implementation.
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path, LockFactory lockFactory)
       throws IOException {
-    return getDirectory(new File(path), lockFactory);
+    return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC);
+  }
+
+  /** Returns the directory instance for the named location.
+   * @param path the path to the directory.
+   * @param lockFactory instance of {@link LockFactory} providing the
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
+   * @param doSync if true (the default), sync() is called
+   *        on all file descriptors before close().  This
+   *        improves the likelihood that the index will
+   *        remain consistent when the OS or machine crashes
+   *        or the power cord is pulled.
+   * @return the FSDirectory for the named file.  */
+  public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync)
+      throws IOException {
+    return getDirectory(new File(path), lockFactory, doSync);
   }
 
   /** Returns the directory instance for the named location.
@@ -154,15 +179,32 @@
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(File file)
     throws IOException {
-    return getDirectory(file, null);
+    return getDirectory(file, null, DEFAULT_DO_SYNC);
   }
 
   /** Returns the directory instance for the named location.
    * @param file the path to the directory.
    * @param lockFactory instance of {@link LockFactory} providing the
-   *        locking implementation.
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(File file, LockFactory lockFactory)
+      throws IOException {
+    return getDirectory(file, lockFactory, DEFAULT_DO_SYNC);
+  }
+
+  /** Returns the directory instance for the named location.
+   * @param file the path to the directory.
+   * @param lockFactory instance of {@link LockFactory} providing the
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
+   * @param doSync if true (the default), sync() is called
+   *        on all file descriptors before close().  This
+   *        improves the likelihood that the index will
+   *        remain consistent when the OS or machine crashes
+   *        or the power cord is pulled.
+   * @return the FSDirectory for the named file.  */
+  public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync)
     throws IOException
   {
     file = new File(file.getCanonicalPath());
@@ -183,7 +225,7 @@
         } catch (Exception e) {
           throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
         }
-        dir.init(file, lockFactory);
+        dir.init(file, lockFactory, doSync);
         DIRECTORIES.put(file, dir);
       } else {
         // Catch the case where a Directory is pulled from the cache, but has a
@@ -254,7 +296,7 @@
 
   protected FSDirectory() {};                     // permit subclassing
 
-  private void init(File path, LockFactory lockFactory) throws IOException {
+  private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException
{
 
     // Set up lockFactory with cascaded defaults: if an instance was passed in,
     // use that; else if locks are disabled, use NoLockFactory; else if the
@@ -262,6 +304,7 @@
     // instantiate that; else, use SimpleFSLockFactory:
 
     directory = path;
+    this.doSync = doSync;
 
     boolean doClearLockID = false;
 
@@ -432,7 +475,7 @@
     if (file.exists() && !file.delete())          // delete existing, if any
       throw new IOException("Cannot overwrite: " + file);
 
-    return new FSIndexOutput(file);
+    return new FSIndexOutput(file, doSync);
   }
 
   // Inherit javadoc
@@ -588,10 +631,13 @@
     // remember if the file is open, so that we don't try to close it
     // more than once
     private boolean isOpen;
-  
-    public FSIndexOutput(File path) throws IOException {
+    private boolean doSync;
+
+    public FSIndexOutput(File path, boolean doSync) throws IOException {
       file = new RandomAccessFile(path, "rw");
+
       isOpen = true;
+      this.doSync = doSync;
     }
   
     /** output methods: */
@@ -601,9 +647,14 @@
     public void close() throws IOException {
       // only close the file if it has not been closed yet
       if (isOpen) {
-        super.close();
-        file.close();
-        isOpen = false;
+        try {
+          super.close();
+          if (doSync)
+            file.getFD().sync();
+        } finally {
+          file.close();
+          isOpen = false;
+        }
       }
     }
   

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestAtomicUpdate.java Sat Nov 10 09:51:00
2007
@@ -177,7 +177,7 @@
     // Second in an FSDirectory:
     String tempDir = System.getProperty("java.io.tmpdir");
     File dirPath = new File(tempDir, "lucene.test.atomic");
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runTest(directory);
     directory.close();
     _TestUtil.rmDir(dirPath);

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat
Nov 10 09:51:00 2007
@@ -152,7 +152,7 @@
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
     IndexSearcher searcher = new IndexSearcher(dir);
     
     Hits hits = searcher.search(new TermQuery(new Term("content", "aaa")));
@@ -172,7 +172,7 @@
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
 
     // open writer
     IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
@@ -232,7 +232,7 @@
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
 
     // make sure searching sees right # hits
     IndexSearcher searcher = new IndexSearcher(dir);
@@ -280,7 +280,7 @@
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
     writer.setUseCompoundFile(doCFS);
     
@@ -311,7 +311,7 @@
       rmDir(outputDir);
 
       try {
-        Directory dir = FSDirectory.getDirectory(fullDir(outputDir));
+        Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false);
 
         boolean autoCommit = 0 == pass;
  

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestCompoundFile.java Sat Nov 10 09:51:00
2007
@@ -62,7 +62,7 @@
        super.setUp();
        File file = new File(System.getProperty("tempDir"), "testIndex");
        _TestUtil.rmDir(file);
-       dir = FSDirectory.getDirectory(file);
+       dir = FSDirectory.getDirectory(file, null, false);
     }
 
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexModifier.java Sat Nov 10 09:51:00
2007
@@ -147,7 +147,7 @@
     if (tempDir == null)
       throw new IOException("java.io.tmpdir undefined, cannot run test");
     File indexDir = new File(tempDir, "lucenetestindex");
-    Directory rd = FSDirectory.getDirectory(indexDir);
+    Directory rd = FSDirectory.getDirectory(indexDir, null, false);
     IndexThread.id = 0;
     IndexThread.idStack.clear();
     IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create);

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestNorms.java Sat Nov 10 09:51:00
2007
@@ -81,7 +81,7 @@
     
     // test with a single index: index1
     File indexDir1 = new File(tempDir, "lucenetestindex1");
-    Directory dir1 = FSDirectory.getDirectory(indexDir1);
+    Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false);
 
     norms = new ArrayList();
     modifiedNorms = new ArrayList();
@@ -99,14 +99,14 @@
     numDocNorms = 0;
     
     File indexDir2 = new File(tempDir, "lucenetestindex2");
-    Directory dir2 = FSDirectory.getDirectory(indexDir2);
+    Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false);
 
     createIndex(dir2);
     doTestNorms(dir2);
 
     // add index1 and index2 to a third index: index3
     File indexDir3 = new File(tempDir, "lucenetestindex3");
-    Directory dir3 = FSDirectory.getDirectory(indexDir3);
+    Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false);
 
     createIndex(dir3);
     IndexWriter iw = new IndexWriter(dir3,anlzr,false);

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing.java Sat Nov 10
09:51:00 2007
@@ -178,7 +178,7 @@
     // FSDir
     String tempDir = System.getProperty("java.io.tmpdir");
     File dirPath = new File(tempDir, "lucene.test.stress");
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, true, null);
     directory.close();
 
@@ -188,7 +188,7 @@
     directory.close();
 
     // With ConcurrentMergeScheduler, in FSDir
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, true, new ConcurrentMergeScheduler());
     directory.close();
 
@@ -198,7 +198,7 @@
     directory.close();
 
     // With ConcurrentMergeScheduler and autoCommit=false, in FSDir
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, false, new ConcurrentMergeScheduler());
     directory.close();
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestThreadedOptimize.java Sat Nov 10
09:51:00 2007
@@ -149,7 +149,7 @@
       throw new IOException("tempDir undefined, cannot run test");
 
     String dirName = tempDir + "/luceneTestThreadedOptimize";
-    directory = FSDirectory.getDirectory(dirName);
+    directory = FSDirectory.getDirectory(dirName, null, false);
     runTest(directory, false, null);
     runTest(directory, true, null);
     runTest(directory, false, new ConcurrentMergeScheduler());

Modified: lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java?rev=593799&r1=593798&r2=593799&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java Sat Nov 10 09:51:00
2007
@@ -320,7 +320,7 @@
     }
 
     public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException
{
-        FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory);
+        FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false);
 
         // First create a 1 doc index:
         IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);



Mime
View raw message