lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1420913 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/ lucene/core/src/test/org/apache/lucene/codecs/compressing/ lucene/test-framework/ lucene/test-framework/src/jav...
Date Wed, 12 Dec 2012 18:51:30 GMT
Author: jpountz
Date: Wed Dec 12 18:51:29 2012
New Revision: 1420913

URL: http://svn.apache.org/viewvc?rev=1420913&view=rev
Log:
LUCENE-4613: CompressingStoredFieldsFormat: fix .abort() when the segment suffix is not empty
(merged from r1420907).

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
Wed Dec 12 18:51:29 2012
@@ -48,16 +48,30 @@ import org.apache.lucene.store.IOContext
 public class CompressingStoredFieldsFormat extends StoredFieldsFormat {
 
   private final String formatName;
+  private final String segmentSuffix;
   private final CompressionMode compressionMode;
   private final int chunkSize;
 
   /**
+   * Create a new {@link CompressingStoredFieldsFormat} with an empty segment 
+   * suffix.
+   * 
+   * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode,
int)
+   */
+  public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode,
int chunkSize) {
+    this(formatName, "", compressionMode, chunkSize);
+  }
+  
+  /**
    * Create a new {@link CompressingStoredFieldsFormat}.
    * <p>
    * <code>formatName</code> is the name of the format. This name will be used
    * in the file formats to perform
    * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec
header checks}.
    * <p>
+   * <code>segmentSuffix</code> is the segment suffix. This suffix is added to

+   * the result file name only if it's not the empty string.
+   * <p>
    * The <code>compressionMode</code> parameter allows you to choose between
    * compression algorithms that have various compression and decompression
    * speeds so that you can pick the one that best fits your indexing and
@@ -81,25 +95,29 @@ public class CompressingStoredFieldsForm
    * @param chunkSize the minimum number of bytes of a single chunk of stored documents
    * @see CompressionMode
    */
-  public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode,
int chunkSize) {
+  public CompressingStoredFieldsFormat(String formatName, String segmentSuffix, 
+                                       CompressionMode compressionMode, int chunkSize) {
     this.formatName = formatName;
+    this.segmentSuffix = segmentSuffix;
     this.compressionMode = compressionMode;
     if (chunkSize < 1) {
       throw new IllegalArgumentException("chunkSize must be >= 1");
     }
     this.chunkSize = chunkSize;
+    
   }
 
   @Override
   public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
       FieldInfos fn, IOContext context) throws IOException {
-    return new CompressingStoredFieldsReader(directory, si, "", fn, context, formatName,
compressionMode);
+    return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, 
+        context, formatName, compressionMode);
   }
 
   @Override
   public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
       IOContext context) throws IOException {
-    return new CompressingStoredFieldsWriter(directory, si, "", context,
+    return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
         formatName, compressionMode, chunkSize);
   }
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
Wed Dec 12 18:51:29 2012
@@ -70,6 +70,7 @@ public final class CompressingStoredFiel
 
   private final Directory directory;
   private final String segment;
+  private final String segmentSuffix;
   private CompressingStoredFieldsIndexWriter indexWriter;
   private IndexOutput fieldsStream;
 
@@ -89,6 +90,7 @@ public final class CompressingStoredFiel
     assert directory != null;
     this.directory = directory;
     this.segment = si.name;
+    this.segmentSuffix = segmentSuffix;
     this.compressionMode = compressionMode;
     this.compressor = compressionMode.newCompressor();
     this.chunkSize = chunkSize;
@@ -287,8 +289,8 @@ public final class CompressingStoredFiel
   public void abort() {
     IOUtils.closeWhileHandlingException(this);
     IOUtils.deleteFilesIgnoringExceptions(directory,
-        IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION),
-        IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
+        IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
+        IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION));
   }
 
   @Override

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
Wed Dec 12 18:51:29 2012
@@ -53,6 +53,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
+import org.junit.Test;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -361,5 +362,41 @@ public class TestCompressingStoredFields
     }
     rd.close();
   }
+  
+  @Test(expected=IllegalArgumentException.class)
+  public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
+    final Document validDoc = new Document();
+    validDoc.add(new IntField("id", 0, Store.YES));
+    iw.addDocument(validDoc);
+    iw.commit();
+    
+    // make sure that #writeField will fail to trigger an abort
+    final Document invalidDoc = new Document();
+    FieldType fieldType = new FieldType();
+    fieldType.setStored(true);
+    invalidDoc.add(new Field("invalid", fieldType) {
+      
+      @Override
+      public String stringValue() {
+        return null;
+      }
+      
+    });
+    
+    try {
+      iw.addDocument(invalidDoc);
+      iw.commit();
+    }
+    finally {
+      int counter = 0;
+      for (String fileName : dir.listAll()) {
+        if (fileName.endsWith(".fdt") || fileName.endsWith(".fdx")) {
+          counter++;
+        }
+      }
+      // Only one .fdt and one .fdx files must have been found
+      assertEquals(2, counter);
+    }
+  }
 
 }

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
Wed Dec 12 18:51:29 2012
@@ -34,30 +34,51 @@ public abstract class CompressingCodec e
   /**
    * Create a random instance.
    */
-  public static CompressingCodec randomInstance(Random random, int chunkSize) {
+  public static CompressingCodec randomInstance(Random random, int chunkSize, boolean withSegmentSuffix)
{
     switch (random.nextInt(4)) {
     case 0:
-      return new FastCompressingCodec(chunkSize);
+      return new FastCompressingCodec(chunkSize, withSegmentSuffix);
     case 1:
-      return new FastDecompressionCompressingCodec(chunkSize);
+      return new FastDecompressionCompressingCodec(chunkSize, withSegmentSuffix);
     case 2:
-      return new HighCompressionCompressingCodec(chunkSize);
+      return new HighCompressionCompressingCodec(chunkSize, withSegmentSuffix);
     case 3:
-      return new DummyCompressingCodec(chunkSize);
+      return new DummyCompressingCodec(chunkSize, withSegmentSuffix);
     default:
       throw new AssertionError();
     }
   }
 
+  /**
+   * Creates a random {@link CompressingCodec} that is using an empty segment 
+   * suffix
+   */
   public static CompressingCodec randomInstance(Random random) {
-    return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500));
+    return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), false);
+  }
+  
+  /**
+   * Creates a random {@link CompressingCodec} that is using a segment suffix
+   */
+  public static CompressingCodec randomInstance(Random random, boolean withSegmentSuffix)
{
+    return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), withSegmentSuffix);
   }
 
   private final CompressingStoredFieldsFormat storedFieldsFormat;
 
-  public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize) {
+  /**
+   * Creates a compressing codec with a given segment suffix
+   */
+  public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode,
int chunkSize) {
     super(name, new Lucene41Codec());
-    this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, compressionMode, chunkSize);
+    this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode,
chunkSize);
+  }
+  
+  /**
+   * Creates a compressing codec with an empty segment suffix
+   */
+  public CompressingCodec(String name, CompressionMode compressionMode, int chunkSize) {
+    this(name, "", compressionMode, chunkSize);
   }
 
   @Override

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/DummyCompressingCodec.java
Wed Dec 12 18:51:29 2012
@@ -82,13 +82,15 @@ public class DummyCompressingCodec exten
   };
 
   /** Constructor that allows to configure the chunk size. */
-  public DummyCompressingCodec(int chunkSize) {
-    super("DummyCompressingStoredFields", DUMMY, chunkSize);
+  public DummyCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+    super("DummyCompressingStoredFields",
+          withSegmentSuffix ? "DummyCompressingStoredFields" : "",
+          DUMMY, chunkSize);
   }
 
   /** Default constructor. */
   public DummyCompressingCodec() {
-    this(1 << 14);
+    this(1 << 14, false);
   }
 
 }

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
Wed Dec 12 18:51:29 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
 public class FastCompressingCodec extends CompressingCodec {
 
   /** Constructor that allows to configure the chunk size. */
-  public FastCompressingCodec(int chunkSize) {
-    super("FastCompressingStoredFields", CompressionMode.FAST, chunkSize);
+  public FastCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+    super("FastCompressingStoredFields", 
+          withSegmentSuffix ? "FastCompressingStoredFields" : "",
+          CompressionMode.FAST, chunkSize);
   }
 
   /** Default constructor. */
   public FastCompressingCodec() {
-    this(1 << 14);
+    this(1 << 14, false);
   }
 
 }

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
Wed Dec 12 18:51:29 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
 public class FastDecompressionCompressingCodec extends CompressingCodec {
 
   /** Constructor that allows to configure the chunk size. */
-  public FastDecompressionCompressingCodec(int chunkSize) {
-    super("FastDecompressionCompressingStoredFields", CompressionMode.FAST_DECOMPRESSION,
chunkSize);
+  public FastDecompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+    super("FastDecompressionCompressingStoredFields",
+          withSegmentSuffix ? "FastDecompressionCompressingStoredFields" : "",
+          CompressionMode.FAST_DECOMPRESSION, chunkSize);
   }
 
   /** Default constructor. */
   public FastDecompressionCompressingCodec() {
-    this(1 << 14);
+    this(1 << 14, false);
   }
 
 }

Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java?rev=1420913&r1=1420912&r2=1420913&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
Wed Dec 12 18:51:29 2012
@@ -21,13 +21,15 @@ package org.apache.lucene.codecs.compres
 public class HighCompressionCompressingCodec extends CompressingCodec {
 
   /** Constructor that allows to configure the chunk size. */
-  public HighCompressionCompressingCodec(int chunkSize) {
-    super("HighCompressionCompressingStoredFields", CompressionMode.HIGH_COMPRESSION, chunkSize);
+  public HighCompressionCompressingCodec(int chunkSize, boolean withSegmentSuffix) {
+    super("HighCompressionCompressingStoredFields",
+          withSegmentSuffix ? "HighCompressionCompressingStoredFields" : "",
+          CompressionMode.HIGH_COMPRESSION, chunkSize);
   }
 
   /** Default constructor. */
   public HighCompressionCompressingCodec() {
-    this(1 << 14);
+    this(1 << 14, false);
   }
 
 }



Mime
View raw message