From commits-return-13438-apmail-jackrabbit-commits-archive=jackrabbit.apache.org@jackrabbit.apache.org Fri Jun 15 09:29:51 2012 Return-Path: X-Original-To: apmail-jackrabbit-commits-archive@www.apache.org Delivered-To: apmail-jackrabbit-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 346A197C9 for ; Fri, 15 Jun 2012 09:29:51 +0000 (UTC) Received: (qmail 74604 invoked by uid 500); 15 Jun 2012 09:29:51 -0000 Delivered-To: apmail-jackrabbit-commits-archive@jackrabbit.apache.org Received: (qmail 74453 invoked by uid 500); 15 Jun 2012 09:29:48 -0000 Mailing-List: contact commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@jackrabbit.apache.org Delivered-To: mailing list commits@jackrabbit.apache.org Received: (qmail 74434 invoked by uid 99); 15 Jun 2012 09:29:47 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jun 2012 09:29:47 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jun 2012 09:29:43 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 98FBE23889D5; Fri, 15 Jun 2012 09:29:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1350552 - in /jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene: AbstractIndex.java CommittableIndexReader.java ReadOnlyIndexReader.java Date: Fri, 15 Jun 2012 09:29:22 -0000 To: commits@jackrabbit.apache.org From: alexparvulescu@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120615092922.98FBE23889D5@eris.apache.org> Author: alexparvulescu Date: Fri Jun 15 09:29:21 2012 New Revision: 1350552 URL: http://svn.apache.org/viewvc?rev=1350552&view=rev Log: JCR-3319 Improve performance of repository updates - tweak index 'deletes' creation Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java?rev=1350552&r1=1350551&r2=1350552&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java Fri Jun 15 09:29:21 2012 @@ -21,7 +21,6 @@ import java.io.OutputStream; import java.io.PrintStream; import java.io.StringReader; import java.util.ArrayList; -import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -307,13 +306,6 @@ abstract class AbstractIndex { } } // if we get here there is no up-to-date read-only reader - // capture snapshot of deleted documents - BitSet deleted = new BitSet(modifiableReader.maxDoc()); - for (int i = 0; i < modifiableReader.maxDoc(); i++) { - if (modifiableReader.isDeleted(i)) { - deleted.set(i); - } - } if (sharedReader == null) { // create new shared reader IndexReader reader = IndexReader.open(getDirectory(), null, true, termInfosIndexDivisor); @@ -321,7 +313,8 @@ abstract class AbstractIndex { reader, cache, initCache); sharedReader = new SharedIndexReader(cr); } - readOnlyReader = new ReadOnlyIndexReader(sharedReader, deleted, modCount); + readOnlyReader = new ReadOnlyIndexReader(sharedReader, + modifiableReader.getDeletedDocs(), modCount); readOnlyReader.acquire(); return readOnlyReader; } Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java?rev=1350552&r1=1350551&r2=1350552&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java Fri Jun 15 09:29:21 2012 @@ -16,11 +16,17 @@ */ package org.apache.jackrabbit.core.query.lucene; +import java.io.IOException; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.CorruptIndexException; - -import java.io.IOException; /** * Wraps an IndexReader and allows to commit changes without @@ -29,6 +35,11 @@ import java.io.IOException; class CommittableIndexReader extends FilterIndexReader { /** + * The maximum size of the delete history. + */ + private static final int DELETE_HISTORY_SIZE = 1000; + + /** * A modification count on this index reader. Initialized with * {@link IndexReader#getVersion()} and incremented with every call to * {@link #doDelete(int)}. @@ -36,6 +47,16 @@ class CommittableIndexReader extends Fil private volatile long modCount; /** + * The history of the most recent deletes. + */ + private final List deleteHistory = new LinkedList(); + + /** + * The deleted docs for this index reader. + */ + private final BitSet deletedDocs = new BitSet(); + + /** * Creates a new CommittableIndexReader based on in. * * @param in the IndexReader to wrap. @@ -43,6 +64,12 @@ class CommittableIndexReader extends Fil CommittableIndexReader(IndexReader in) { super(in); modCount = in.getVersion(); + int maxDocs = in.maxDoc(); + for (int i = 0; i < maxDocs; i++) { + if (in.isDeleted(i)) { + deletedDocs.set(i); + } + } } //------------------------< FilterIndexReader >----------------------------- @@ -55,6 +82,11 @@ class CommittableIndexReader extends Fil protected void doDelete(int n) throws CorruptIndexException, IOException { super.doDelete(n); modCount++; + if (deleteHistory.size() >= DELETE_HISTORY_SIZE) { + deleteHistory.remove(0); + } + deleteHistory.add(n); + deletedDocs.set(n); } //------------------------< additional methods >---------------------------- @@ -65,4 +97,44 @@ class CommittableIndexReader extends Fil long getModificationCount() { return modCount; } + + /** + * Returns the document numbers of deleted nodes since the given + * modCount. + * + * @param modCount a modification count. + * @return document numbers of deleted nodes or null if this + * index reader cannot provide those document number. e.g. modCount + * is too far back in the past. + * @throws IllegalArgumentException if modCount is larger than + * {@link #getModificationCount()}. + */ + Collection getDeletedSince(long modCount) + throws IllegalArgumentException { + if (modCount > this.modCount) { + throw new IllegalArgumentException("modCount: " + + modCount + " > " + this.modCount); + } + if (modCount == this.modCount) { + return Collections.emptyList(); + } + long num = this.modCount - modCount; + if (num > deleteHistory.size()) { + return null; + } + List deletes = new ArrayList((int) num); + for (Integer d : deleteHistory.subList((int) (deleteHistory.size() - num), + deleteHistory.size())) { + deletes.add(d); + } + return deletes; + } + + /** + * Returns a copy of the deleted documents BitSet. + * @return the deleted documents of this index reader. + */ + BitSet getDeletedDocs() { + return (BitSet) deletedDocs.clone(); + } } Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java?rev=1350552&r1=1350551&r2=1350552&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java Fri Jun 15 09:29:21 2012 @@ -22,6 +22,7 @@ import org.apache.lucene.index.TermPosit import java.io.IOException; import java.util.BitSet; +import java.util.Collection; import java.util.Map; /** @@ -36,7 +37,7 @@ class ReadOnlyIndexReader extends RefCou * The deleted documents as initially read from the IndexReader passed * in the constructor of this class. */ - private final BitSet deleted; + private BitSet deleted; /** * The version of the index reader from where the deleted BitSet was @@ -93,10 +94,14 @@ class ReadOnlyIndexReader extends RefCou * info. */ void updateDeletedDocs(CommittableIndexReader reader) { - int maxDoc = reader.maxDoc(); - for (int i = 0; i < maxDoc; i++) { - if (reader.isDeleted(i)) { - deleted.set(i); + Collection deletes = reader.getDeletedSince(deletedDocsVersion); + if (deletes == null) { + // full update needed + this.deleted = reader.getDeletedDocs(); + } else { + // incremental update + for (Integer d : deletes) { + deleted.set(d); } } deletedDocsVersion = reader.getModificationCount();