lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1594566 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/CHANGES.txt lucene/core/ lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java lucene/core/src/test/org/apache/lucene/index/dvupdates.48.zip
Date Wed, 14 May 2014 12:45:33 GMT
Author: shaie
Date: Wed May 14 12:45:33 2014
New Revision: 1594566

URL: http://svn.apache.org/r1594566
Log:
LUCENE-5619: add back-compat index+test for doc-values updates

Added:
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/dvupdates.48.zip
      - copied unchanged from r1594561, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/dvupdates.48.zip
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (contents, props changed)

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1594566&r1=1594565&r2=1594566&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed May 14 12:45:33 2014
@@ -120,6 +120,9 @@ Test Framework
 
 * LUCENE-5622: Fail tests if they print over the given limit of bytes to 
   System.out or System.err. (Robert Muir, Dawid Weiss)
+  
+* LUCENE-5619: Added backwards compatibility tests to ensure we can update existing
+  indexes with doc-values updates. (Shai Erera, Robert Muir)
 
 ======================= Lucene 4.8.0 =======================
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1594566&r1=1594565&r2=1594566&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
Wed May 14 12:45:33 2014
@@ -162,6 +162,64 @@ public class TestBackwardsCompatibility 
   }
   */
   
+  private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value)
throws IOException {
+    writer.updateNumericDocValue(new Term("id", id), f, value);
+    writer.updateNumericDocValue(new Term("id", id), cf, value*2);
+  }
+  
+  private void updateBinary(IndexWriter writer, String id, String f, String cf, long value)
throws IOException {
+    writer.updateBinaryDocValue(new Term("id", id), f, TestBinaryDocValuesUpdates.toBytes(value));
+    writer.updateBinaryDocValue(new Term("id", id), cf, TestBinaryDocValuesUpdates.toBytes(value*2));
+  }
+
+/*  // Creates an index with DocValues updates
+  public void testCreateIndexWithDocValuesUpdates() throws Exception {
+    // we use a real directory name that is not cleaned up,
+    // because this method is only used to create backwards
+    // indexes:
+    File indexDir = new File("/tmp/idx/dvupdates");
+    TestUtil.rm(indexDir);
+    Directory dir = newFSDirectory(indexDir);
+    
+    IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
+      .setUseCompoundFile(false).setMergePolicy(NoMergePolicy.INSTANCE);
+    IndexWriter writer = new IndexWriter(dir, conf);
+    // create an index w/ few doc-values fields, some with updates and some without
+    for (int i = 0; i < 30; i++) {
+      Document doc = new Document();
+      doc.add(new StringField("id", "" + i, Store.NO));
+      doc.add(new NumericDocValuesField("ndv1", i));
+      doc.add(new NumericDocValuesField("ndv1_c", i*2));
+      doc.add(new NumericDocValuesField("ndv2", i*3));
+      doc.add(new NumericDocValuesField("ndv2_c", i*6));
+      doc.add(new BinaryDocValuesField("bdv1", TestBinaryDocValuesUpdates.toBytes(i)));
+      doc.add(new BinaryDocValuesField("bdv1_c", TestBinaryDocValuesUpdates.toBytes(i*2)));
+      doc.add(new BinaryDocValuesField("bdv2", TestBinaryDocValuesUpdates.toBytes(i*3)));
+      doc.add(new BinaryDocValuesField("bdv2_c", TestBinaryDocValuesUpdates.toBytes(i*6)));
+      writer.addDocument(doc);
+      if ((i+1) % 10 == 0) {
+        writer.commit(); // flush every 10 docs
+      }
+    }
+    
+    // first segment: no updates
+    
+    // second segment: update two fields, same gen
+    updateNumeric(writer, "10", "ndv1", "ndv1_c", 100L);
+    updateBinary(writer, "11", "bdv1", "bdv1_c", 100L);
+    writer.commit();
+    
+    // third segment: update few fields, different gens, few docs
+    updateNumeric(writer, "20", "ndv1", "ndv1_c", 100L);
+    updateBinary(writer, "21", "bdv1", "bdv1_c", 100L);
+    writer.commit();
+    updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again
+    writer.commit();
+    
+    writer.close();
+    dir.close();
+  }*/
+
   final static String[] oldNames = {"40.cfs",
                                     "40.nocfs",
                                     "41.cfs",
@@ -975,4 +1033,62 @@ public class TestBackwardsCompatibility 
     TestUtil.checkIndex(dir);
     dir.close();
   }
+
+  public static final String dvUpdatesIndex = "dvupdates.48.zip";
+
+  private void assertNumericDocValues(AtomicReader r, String f, String cf) throws IOException
{
+    NumericDocValues ndvf = r.getNumericDocValues(f);
+    NumericDocValues ndvcf = r.getNumericDocValues(cf);
+    for (int i = 0; i < r.maxDoc(); i++) {
+      assertEquals(ndvcf.get(i), ndvf.get(i)*2);
+    }
+  }
+  
+  private void assertBinaryDocValues(AtomicReader r, String f, String cf) throws IOException
{
+    BinaryDocValues bdvf = r.getBinaryDocValues(f);
+    BinaryDocValues bdvcf = r.getBinaryDocValues(cf);
+    BytesRef scratch = new BytesRef();
+    for (int i = 0; i < r.maxDoc(); i++) {
+      assertEquals(TestBinaryDocValuesUpdates.getValue(bdvcf, i, scratch ), TestBinaryDocValuesUpdates.getValue(bdvf,
i, scratch)*2);
+    }
+  }
+  
+  private void verifyDocValues(Directory dir) throws IOException {
+    DirectoryReader reader = DirectoryReader.open(dir);
+    for (AtomicReaderContext context : reader.leaves()) {
+      AtomicReader r = context.reader();
+      assertNumericDocValues(r, "ndv1", "ndv1_c");
+      assertNumericDocValues(r, "ndv2", "ndv2_c");
+      assertBinaryDocValues(r, "bdv1", "bdv1_c");
+      assertBinaryDocValues(r, "bdv2", "bdv2_c");
+    }
+    reader.close();
+  }
+  
+  public void testDocValuesUpdates() throws Exception {
+    File oldIndexDir = createTempDir("dvupdates");
+    TestUtil.unzip(getDataFile(dvUpdatesIndex), oldIndexDir);
+    Directory dir = newFSDirectory(oldIndexDir);
+    
+    verifyDocValues(dir);
+    
+    // update fields and verify index
+    IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    IndexWriter writer = new IndexWriter(dir, conf);
+    updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);
+    updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);
+    updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);
+    updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);
+    writer.commit();
+    verifyDocValues(dir);
+    
+    // merge all segments
+    writer.forceMerge(1);
+    writer.commit();
+    verifyDocValues(dir);
+    
+    writer.close();
+    dir.close();
+  }
+  
 }



Mime
View raw message