Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3540F1036F for ; Mon, 5 Jan 2015 18:06:23 +0000 (UTC) Received: (qmail 12974 invoked by uid 500); 5 Jan 2015 18:06:24 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 12964 invoked by uid 99); 5 Jan 2015 18:06:24 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 Jan 2015 18:06:24 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id E2F10AC003F; Mon, 5 Jan 2015 18:06:23 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1649601 - in /lucene/dev/branches/lucene_solr_4_10: ./ lucene/ lucene/CHANGES.txt lucene/core/ lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java Date: Mon, 05 Jan 2015 18:06:23 -0000 To: commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150105180623.E2F10AC003F@hades.apache.org> Author: mikemccand Date: Mon Jan 5 18:06:23 2015 New Revision: 1649601 URL: http://svn.apache.org/r1649601 Log: LUCENE-6161: reuse DocsEnum when resolving deleted terms/queries to doc id Modified: lucene/dev/branches/lucene_solr_4_10/ (props changed) lucene/dev/branches/lucene_solr_4_10/lucene/ (props changed) lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt lucene/dev/branches/lucene_solr_4_10/lucene/core/ (props changed) lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java Modified: lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt?rev=1649601&r1=1649600&r2=1649601&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (original) +++ lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt Mon Jan 5 18:06:23 2015 @@ -10,6 +10,10 @@ Bug fixes * LUCENE-6019, LUCENE-6117: Remove -Dtests.assert to make IndexWriter infoStream sane. (Robert Muir, Mike McCandless) +* LUCENE-6161: Resolving deletes was failing to reuse DocsEnum likely + causing substantial performance cost for use cases that frequently + delete old documents (Mike McCandless) + ======================= Lucene 4.10.3 ====================== Bug fixes Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java?rev=1649601&r1=1649600&r2=1649601&view=diff ============================================================================== --- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java (original) +++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java Mon Jan 5 18:06:23 2015 @@ -393,7 +393,7 @@ class BufferedUpdatesStream implements A TermsEnum termsEnum = null; String currentField = null; - DocsEnum docs = null; + DocsEnum docsEnum = null; assert checkDeleteTerm(null); @@ -416,36 +416,38 @@ class BufferedUpdatesStream implements A } if (termsEnum == null) { + // no terms in this field continue; } + assert checkDeleteTerm(term); // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes())) { // we don't need term frequencies for this - DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE); + docsEnum = termsEnum.docs(rld.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); //System.out.println("BDS: got docsEnum=" + docsEnum); - if (docsEnum != null) { - while (true) { - final int docID = docsEnum.nextDoc(); - //System.out.println(Thread.currentThread().getName() + " del term=" + term + " doc=" + docID); - if (docID == DocIdSetIterator.NO_MORE_DOCS) { - break; - } - if (!any) { - rld.initWritableLiveDocs(); - any = true; - } - // NOTE: there is no limit check on the docID - // when deleting by Term (unlike by Query) - // because on flush we apply all Term deletes to - // each segment. So all Term deleting here is - // against prior segments: - if (rld.delete(docID)) { - delCount++; - } + assert docsEnum != null; + + while (true) { + final int docID = docsEnum.nextDoc(); + //System.out.println(Thread.currentThread().getName() + " del term=" + term + " doc=" + docID); + if (docID == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + if (!any) { + rld.initWritableLiveDocs(); + any = true; + } + // NOTE: there is no limit check on the docID + // when deleting by Term (unlike by Query) + // because on flush we apply all Term deletes to + // each segment. So all Term deleting here is + // against prior segments: + if (rld.delete(docID)) { + delCount++; } } } @@ -475,7 +477,7 @@ class BufferedUpdatesStream implements A String currentField = null; TermsEnum termsEnum = null; - DocsEnum docs = null; + DocsEnum docsEnum = null; //System.out.println(Thread.currentThread().getName() + " numericDVUpdate reader=" + reader); for (DocValuesUpdate update : updates) { @@ -501,19 +503,19 @@ class BufferedUpdatesStream implements A termsEnum = terms.iterator(termsEnum); } else { termsEnum = null; - continue; // no terms in that field } } if (termsEnum == null) { + // no terms in this field continue; } + // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes())) { // we don't need term frequencies for this - DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE); - + docsEnum = termsEnum.docs(rld.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); //System.out.println("BDS: got docsEnum=" + docsEnum); DocValuesFieldUpdates dvUpdates = dvUpdatesContainer.getUpdates(update.field, update.type);