Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
Reply-To: java-dev@lucene.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r770841 - in /lucene/java/trunk: CHANGES.txt
 src/java/org/apache/lucene/store/FileSwitchDirectory.java
 src/test/org/apache/lucene/index/TestIndexWriterReader.java
 src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
Date: Fri, 01 May 2009 22:12:25 -0000
To: java-commits@lucene.apache.org
From: mikemccand@apache.org
Message-Id: <20090501221225.8688323888A2@eris.apache.org>

Author: mikemccand
Date: Fri May  1 22:12:24 2009
New Revision: 770841

URL: http://svn.apache.org/viewvc?rev=770841&view=rev
Log:
LUCENE-1618: added FileSwitchDirectory, which will be used for NRT readers in IndexWriter

Added:
    lucene/java/trunk/src/java/org/apache/lucene/store/FileSwitchDirectory.java
    lucene/java/trunk/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterReader.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=770841&r1=770840&r2=770841&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Fri May  1 22:12:24 2009
@@ -282,6 +282,14 @@
 
 22. LUCENE-1605: Added BitVector.subset().  (Jeremy Volkman via Mike
     McCandless)
+    
+23. LUCENE-1618: Added FileSwitchDirectory that enables files with
+    specified extensions to be stored in a primary directory and the
+    rest of the files to be stored in the secondary directory.  For
+    example, this can be useful for the large doc-store (stored
+    fields, term vectors) files in FSDirectory and the rest of the
+    index files in a RAMDirectory. (Jason Rutherglen via Mike
+    McCandless)
 
 23. LUCENE-1494: Added FieldMaskingSpanQuery which can be used to
     cross-correlate Spans from different fields.

Added: lucene/java/trunk/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=770841&view=auto
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/FileSwitchDirectory.java (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/FileSwitchDirectory.java Fri May  1 22:12:24 2009
@@ -0,0 +1,127 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Files with the specified extensions are placed in the
+ * primary directory; others are placed in the secondary
+ * directory.  The provided Set must not change once passed
+ * to this class, and must allow multiple threads to call
+ * contains at once.
+ */
+
+public class FileSwitchDirectory extends Directory {
+  private final Directory secondaryDir;
+  private final Directory primaryDir;
+  private final Set primaryExtensions;
+  private boolean doClose;
+
+  public FileSwitchDirectory(Set primaryExtensions, Directory primaryDir, Directory secondaryDir, boolean doClose) {
+    this.primaryExtensions = primaryExtensions;
+    this.primaryDir = primaryDir;
+    this.secondaryDir = secondaryDir;
+    this.doClose = doClose;
+    this.lockFactory = primaryDir.getLockFactory();
+  }
+  
+  public void close() throws IOException {
+    if (doClose) {
+      try {
+        secondaryDir.close();
+      } finally { 
+        primaryDir.close();
+      }
+      doClose = false;
+    }
+  }
+  
+  public String[] listAll() throws IOException {
+    List list = new ArrayList();
+    String[] ramFiles = secondaryDir.listAll();
+    for (int x = 0; x < ramFiles.length; x++) {
+      list.add(ramFiles[x]);
+    }
+    String[] fsFiles = primaryDir.listAll();
+    for (int x = 0; x < fsFiles.length; x++) {
+      list.add(fsFiles[x]);
+    }
+    return (String[]) list.toArray(new String[0]);
+  }
+  
+  public String[] list() throws IOException {
+    return listAll();
+  }
+
+  public static String getExtension(String name) {
+    int i = name.lastIndexOf('.');
+    if (i == -1) {
+      return "";
+    }
+    return name.substring(i+1, name.length());
+  }
+
+  private Directory getDirectory(String name) {
+    String ext = getExtension(name);
+    if (primaryExtensions.contains(ext)) {
+      return primaryDir;
+    } else {
+      return secondaryDir;
+    }
+  }
+
+  public boolean fileExists(String name) throws IOException {
+    return getDirectory(name).fileExists(name);
+  }
+
+  public long fileModified(String name) throws IOException {
+    return getDirectory(name).fileModified(name);
+  }
+
+  public void touchFile(String name) throws IOException {
+    getDirectory(name).touchFile(name);
+  }
+
+  public void deleteFile(String name) throws IOException {
+    getDirectory(name).deleteFile(name);
+  }
+
+  public void renameFile(String from, String to) throws IOException {
+    getDirectory(from).renameFile(from, to);
+  }
+
+  public long fileLength(String name) throws IOException {
+    return getDirectory(name).fileLength(name);
+  }
+
+  public IndexOutput createOutput(String name) throws IOException {
+    return getDirectory(name).createOutput(name);
+  }
+
+  public void sync(String name) throws IOException {
+    getDirectory(name).sync(name);
+  }
+
+  public IndexInput openInput(String name) throws IOException {
+    return getDirectory(name).openInput(name);
+  }
+}

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=770841&r1=770840&r2=770841&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterReader.java Fri May  1 22:12:24 2009
@@ -537,7 +537,6 @@
 
   public static void createIndexNoClose(boolean multiSegment, String indexName,
       IndexWriter w) throws IOException {
-    w.setMergePolicy(new LogDocMergePolicy());
     for (int i = 0; i < 100; i++) {
       w.addDocument(createDocument(i, indexName, 4));
     }

Added: lucene/java/trunk/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java?rev=770841&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java Fri May  1 22:12:24 2009
@@ -0,0 +1,71 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.TestIndexWriterReader;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestFileSwitchDirectory extends LuceneTestCase {
+  /**
+   * Test if writing doc stores to disk and everything else to ram works.
+   * @throws IOException
+   */
+  public void testBasic() throws IOException {
+    Set fileExtensions = new HashSet();
+    fileExtensions.add("fdt");
+    fileExtensions.add("fdx");
+    
+    Directory primaryDir = new MockRAMDirectory();
+    RAMDirectory secondaryDir = new MockRAMDirectory();
+    
+    FileSwitchDirectory fsd = new FileSwitchDirectory(fileExtensions, primaryDir, secondaryDir, true);
+    IndexWriter writer = new IndexWriter(fsd, new WhitespaceAnalyzer(),
+        IndexWriter.MaxFieldLength.LIMITED);
+    writer.setUseCompoundFile(false);
+    TestIndexWriterReader.createIndexNoClose(true, "ram", writer);
+    IndexReader reader = writer.getReader();
+    assertEquals(100, reader.maxDoc());
+    writer.commit();
+    // we should see only fdx,fdt files here
+    String[] files = primaryDir.listAll();
+    assertTrue(files.length > 0);
+    for (int x=0; x < files.length; x++) {
+      String ext = FileSwitchDirectory.getExtension(files[x]);
+      assertTrue(fileExtensions.contains(ext));
+    }
+    files = secondaryDir.listAll();
+    assertTrue(files.length > 0);
+    // we should not see fdx,fdt files here
+    for (int x=0; x < files.length; x++) {
+      String ext = FileSwitchDirectory.getExtension(files[x]);
+      assertFalse(fileExtensions.contains(ext));
+    }
+    reader.close();
+    writer.close();
+    
+    primaryDir.close();
+    secondaryDir.close();
+  }
+}