lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r615160 - in /lucene/java/trunk: CHANGES.txt src/java/org/apache/lucene/index/CompoundFileWriter.java src/java/org/apache/lucene/store/FSDirectory.java src/java/org/apache/lucene/store/IndexOutput.java
Date Fri, 25 Jan 2008 09:42:30 GMT
Author: mikemccand
Date: Fri Jan 25 01:42:29 2008
New Revision: 615160

URL: http://svn.apache.org/viewvc?rev=615160&view=rev
Log:
LUCENE-705: pre-set the CFS file length before building it

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/index/CompoundFileWriter.java
    lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
    lucene/java/trunk/src/java/org/apache/lucene/store/IndexOutput.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=615160&r1=615159&r2=615160&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Fri Jan 25 01:42:29 2008
@@ -20,6 +20,13 @@
 
 Optimizations
 
+ 1. LUCENE-705: When building a compound file, use
+    RandomAccessFile.setLength() to tell the OS/filesystem to
+    pre-allocate space for the file.  This may improve fragmentation
+    in how the CFS file is stored, and allows us to detect an upcoming
+    disk full situation before actually filling up the disk.  (Mike
+    McCandless)
+
 Documentation
 
 Build

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/CompoundFileWriter.java?rev=615160&r1=615159&r2=615160&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/CompoundFileWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/CompoundFileWriter.java Fri Jan 25
01:42:29 2008
@@ -156,13 +156,24 @@
             // Remember the positions of directory entries so that we can
             // adjust the offsets later
             Iterator it = entries.iterator();
+            long totalSize = 0;
             while(it.hasNext()) {
                 FileEntry fe = (FileEntry) it.next();
                 fe.directoryOffset = os.getFilePointer();
                 os.writeLong(0);    // for now
                 os.writeString(fe.file);
+                totalSize += directory.fileLength(fe.file);
             }
 
+            // Pre-allocate size of file as optimization --
+            // this can potentially help IO performance as
+            // we write the file and also later during
+            // searching.  It also uncovers a disk-full
+            // situation earlier and hopefully without
+            // actually filling disk to 100%:
+            final long finalLength = totalSize+os.getFilePointer();
+            os.setLength(finalLength);
+
             // Open the files and copy their data into the stream.
             // Remember the locations of each file's data section.
             byte buffer[] = new byte[16384];
@@ -180,6 +191,8 @@
                 os.seek(fe.directoryOffset);
                 os.writeLong(fe.dataOffset);
             }
+
+            assert finalLength == os.length();
 
             // Close the output stream. Set the os to null before trying to
             // close so that if an exception occurs during the close, the

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java?rev=615160&r1=615159&r2=615160&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java Fri Jan 25 01:42:29
2008
@@ -615,6 +615,8 @@
     public long length() throws IOException {
       return file.length();
     }
-  
+    public void setLength(long length) throws IOException {
+      file.setLength(length);
+    }
   }
 }

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/IndexOutput.java?rev=615160&r1=615159&r2=615160&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/IndexOutput.java Fri Jan 25 01:42:29
2008
@@ -189,5 +189,15 @@
   /** The number of bytes in the file. */
   public abstract long length() throws IOException;
 
-
+  /** Set the file length. By default, this method does
+   * nothing (it's optional for a Directory to implement
+   * it).  But, certain Directory implementations (for
+   * example @see FSDirectory) can use this to inform the
+   * underlying IO system to pre-allocate the file to the
+   * specified size.  If the length is longer than the
+   * current file length, the bytes added to the file are
+   * undefined.  Otherwise the file is truncated.
+   * @param length file length
+   */
+  public void setLength(long length) throws IOException {};
 }



Mime
View raw message