lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject svn commit: r805774 - in /lucene/solr/trunk: ./ src/common/org/apache/solr/common/params/ src/java/org/apache/solr/handler/ src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/ src/test/org/apache/solr/update/
Date Wed, 19 Aug 2009 12:21:22 GMT
Author: noble
Date: Wed Aug 19 12:21:22 2009
New Revision: 805774

URL: http://svn.apache.org/viewvc?rev=805774&view=rev
Log:
SOLR-1275: Add expungeDeletes to DirectUpdateHandler2

Modified:
    lucene/solr/trunk/CHANGES.txt
    lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
    lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
    lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
    lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
    lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
    lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
    lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
    lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java

Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
@@ -267,7 +267,9 @@
     HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, HTMLStripCharFilter
can be used
     with an arbitrary Tokenizer. (koji)
 
-68. SOLR-1367: Added callback mechanism for converting DocList to SolrDocumentList in SolrPluginUtils
(gsingers)  
+68. SOLR-1367: Added callback mechanism for converting DocList to SolrDocumentList in SolrPluginUtils
(gsingers)
+
+68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
 
 
 Optimizations

Modified: lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java (original)
+++ lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java Wed Aug 19
12:21:22 2009
@@ -49,4 +49,6 @@
    * If optimizing, set the maximum number of segments left in the index after optimization.
 1 is the default (and is equivalent to calling IndexWriter.optimize() in Lucene).
    */
   public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
+
+  public static final String EXPUNGE_DELETES = "expungeDeletes";
 }

Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java Wed Aug 19
12:21:22 2009
@@ -68,6 +68,7 @@
       CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
       cmd.waitFlush    = params.getBool( UpdateParams.WAIT_FLUSH,    cmd.waitFlush    );
       cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
+      cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
       cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
       req.getCore().getUpdateHandler().commit( cmd );
       
@@ -101,6 +102,7 @@
       CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
       cmd.waitFlush    = params.getBool( UpdateParams.WAIT_FLUSH,    cmd.waitFlush    );
       cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
+      cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
     
       cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
       processor.processCommit( cmd );
       return true;

Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed Aug 19 12:21:22
2009
@@ -155,6 +155,8 @@
                 sawWaitSearcher = true;
               } else if (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
                 cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
+              } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
+                cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
               } else {
                 XmlUpdateRequestHandler.log.warn("unexpected attribute commit/@" + attrName);
               }

Modified: lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java Wed Aug 19
12:21:22 2009
@@ -23,6 +23,7 @@
   public boolean optimize;
   public boolean waitFlush;
   public boolean waitSearcher=true;
+  public boolean expungeDeletes = false;
 
   /**
    * During optimize, optimize down to <= this many segments.  Must be >= 1
@@ -39,6 +40,7 @@
     return "commit(optimize="+optimize
             +",waitFlush="+waitFlush
             +",waitSearcher="+waitSearcher
+            +",expungeDeletes="+expungeDeletes
             +')';
   }
 }

Modified: lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java Wed Aug 19
12:21:22 2009
@@ -241,9 +241,10 @@
     synchronized (this) {
       pset.clear();
       closeSearcher();  // flush any deletes
-      if (cmd.optimize) {
+      if (cmd.optimize || cmd.expungeDeletes) {
         openWriter();  // writer needs to be open to optimize
-        writer.optimize(cmd.maxOptimizeSegments);
+        if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
+        if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
       }
       closeWriter();
 

Modified: lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java Wed Aug 19
12:21:22 2009
@@ -130,6 +130,7 @@
   AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
   AtomicLong deleteByQueryCommands= new AtomicLong();
   AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
+  AtomicLong expungeDeleteCommands = new AtomicLong();
   AtomicLong mergeIndexesCommands = new AtomicLong();
   AtomicLong commitCommands= new AtomicLong();
   AtomicLong optimizeCommands= new AtomicLong();
@@ -382,6 +383,8 @@
 
     if (cmd.optimize) {
       optimizeCommands.incrementAndGet();
+    } else if (cmd.expungeDeletes) {
+      expungeDeleteCommands.incrementAndGet();
     } else {
       commitCommands.incrementAndGet();
     }
@@ -402,6 +405,10 @@
       }
 
       closeWriter();
+      if (!cmd.optimize && cmd.expungeDeletes) {
+        openWriter();
+        writer.expungeDeletes();
+      }
 
       callPostCommitCallbacks();
       if (cmd.optimize) {

Modified: lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java (original)
+++ lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java Wed
Aug 19 12:21:22 2009
@@ -153,6 +153,12 @@
     return this;
   }
 
+  public UpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher,
int maxSegments , boolean expungeDeletes) {
+    setAction(action, waitFlush, waitSearcher,maxSegments) ;
+    params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
+    return this;
+  }
+
   /**
    * @since Solr 1.4
    */

Modified: lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java Wed Aug
19 12:21:22 2009
@@ -17,20 +17,32 @@
 
 package org.apache.solr.update;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.SolrIndexReader;
+import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.RefCounted;
 
 /**
  * 
@@ -247,6 +259,90 @@
             );
   }
   
+  public void testExpungeDeletes() throws Exception {
+    for (int x = 0; x < 3000; x++) {
+      addSimpleDoc(x + "");
+    }
+    SolrCore core = h.getCore();
+    UpdateHandler updater = core.getUpdateHandler();
+    CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
+    cmtCmd.waitSearcher = true;
+    updater.commit(cmtCmd);
+
+    List<String> todelete = new ArrayList<String>();
+
+    Set<String> segsdel = new HashSet<String>();
+
+    SegmentReader[] sirs = getSegmentReaders(core);
+    assertTrue(sirs.length > 6);
+    todelete.add(getNthIDTerm(2, sirs[0]));
+    segsdel.add(sirs[0].getSegmentName());
+    
+    todelete.add(getNthIDTerm(7, sirs[2]));
+    segsdel.add(sirs[2].getSegmentName());
+    
+    todelete.add(getNthIDTerm(4, sirs[5]));
+    segsdel.add(sirs[5].getSegmentName());
+    
+    for (String id : todelete) {
+      deleteSimpleDoc(id);
+    }
+    // commit the deletes
+    cmtCmd = new CommitUpdateCommand(false);
+    cmtCmd.waitSearcher = true;
+    updater.commit(cmtCmd);
+    
+    // expunge deletes
+    cmtCmd = new CommitUpdateCommand(false);
+    cmtCmd.waitSearcher = true;
+    cmtCmd.expungeDeletes = true;
+    updater.commit(cmtCmd);
+    
+    // we'll have fewer segments
+    SegmentReader[] sirs2 = getSegmentReaders(core);
+    assertTrue(sirs.length > sirs2.length);
+    // check the actual segment names
+    for (SegmentReader sr : sirs2) {
+      assertTrue(!segsdel.contains(sr.getSegmentName()));
+    }
+  }
+
+  SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
+    RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, null);
+    SolrIndexSearcher is = ref.get();
+    SegmentReader[] segmentReaders = null;
+    try {
+      SolrIndexReader reader = is.getReader();
+      IndexReader[] subreaders = reader.getSequentialSubReaders();
+      segmentReaders = new SegmentReader[subreaders.length];
+      for (int x = 0; x < subreaders.length; x++) {
+        assert subreaders[x] instanceof SolrIndexReader;
+        SolrIndexReader sir = (SolrIndexReader) subreaders[x];
+        SegmentReader sr = (SegmentReader) sir.getWrappedReader();
+        segmentReaders[x] = sr;
+      }
+    } finally {
+      ref.decref();
+    }
+    return segmentReaders;
+  }
+
+  private String getNthIDTerm(int n, IndexReader r) throws IOException {
+    TermEnum te = r.terms(new Term("id", ""));
+    try {
+      int x = 0;
+      do {
+        if (x >= n) {
+          return te.term().text();
+        }
+        x++;
+      } while (te.next());
+    } finally {
+      te.close();
+    }
+    return null;
+  }
+  
   private void addSimpleDoc(String id) throws Exception {
     SolrCore core = h.getCore();
     



Mime
View raw message