Return-Path: X-Original-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Delivered-To: apmail-jackrabbit-oak-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 94CED1048D for ; Tue, 1 Apr 2014 20:10:45 +0000 (UTC) Received: (qmail 24530 invoked by uid 500); 1 Apr 2014 20:10:45 -0000 Delivered-To: apmail-jackrabbit-oak-commits-archive@jackrabbit.apache.org Received: (qmail 23890 invoked by uid 500); 1 Apr 2014 20:10:42 -0000 Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: oak-dev@jackrabbit.apache.org Delivered-To: mailing list oak-commits@jackrabbit.apache.org Received: (qmail 23800 invoked by uid 99); 1 Apr 2014 20:10:40 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Apr 2014 20:10:40 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Apr 2014 20:10:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8B0152388868; Tue, 1 Apr 2014 20:10:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1583771 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/ oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/ oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/ oak-lucen... Date: Tue, 01 Apr 2014 20:10:18 -0000 To: oak-commits@jackrabbit.apache.org From: alexparvulescu@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140401201018.8B0152388868@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: alexparvulescu Date: Tue Apr 1 20:10:17 2014 New Revision: 1583771 URL: http://svn.apache.org/r1583771 Log: OAK-1654 Composite index aggregates Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java?rev=1583771&r1=1583770&r2=1583771&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java Tue Apr 1 20:10:17 2014 @@ -16,15 +16,15 @@ */ package org.apache.jackrabbit.oak.plugins.index.aggregate; -import java.util.ArrayList; + import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import org.apache.jackrabbit.oak.api.PropertyValue; -import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd; import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression; import org.apache.jackrabbit.oak.query.fulltext.FullTextOr; @@ -32,14 +32,17 @@ import org.apache.jackrabbit.oak.query.f import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor; import org.apache.jackrabbit.oak.query.index.FilterImpl; import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.query.Cursors; import org.apache.jackrabbit.oak.spi.query.Cursors.AbstractCursor; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.IndexRow; import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex; import org.apache.jackrabbit.oak.spi.state.NodeState; +import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; /** * A virtual full-text that can aggregate nodes based on aggregate definitions. @@ -58,7 +61,13 @@ public class AggregateIndex implements F if (baseIndex == null) { return Double.POSITIVE_INFINITY; } - return baseIndex.getCost(filter, rootState) - 0.05; + double localCost = Double.POSITIVE_INFINITY; + FullTextExpression e = filter.getFullTextConstraint(); + if (e != null && hasCompositeExpression(e)) { + localCost = flattenCost(e, filter, baseIndex, rootState); + } + double baseCost = baseIndex.getCost(filter, rootState); + return Math.min(localCost, baseCost) - 0.05; } @Override @@ -67,70 +76,172 @@ public class AggregateIndex implements F if (baseIndex.getNodeAggregator() == null) { return baseIndex.query(filter, rootState); } - return new AggregationCursor(baseIndex.query( - newAggregationFilter(filter), rootState), - baseIndex.getNodeAggregator(), rootState); + return newCursor(filter, baseIndex, rootState); } - private static Filter newAggregationFilter(Filter filter) { - FilterImpl f = new FilterImpl(filter); - // disables node type checks for now - f.setMatchesAllTypes(true); + private static Cursor newCursor(Filter f, FulltextQueryIndex index, + NodeState state) { + FullTextExpression e = f.getFullTextConstraint(); + if (hasCompositeExpression(e)) { + Cursor c = flatten(e, f, index, state); + if (c != null) { + return c; + } + } + return new AggregationCursor(index.query(newAggregationFilter(f, null), + state), index.getNodeAggregator(), state); + } + + private static boolean hasCompositeExpression(FullTextExpression ft) { + if (ft == null) { + return false; + } + final AtomicReference composite = new AtomicReference(); + composite.set(false); - // TODO OAK-828 - // FullTextExpression constraint = filter.getFullTextConstraint(); - // constraint = getFlatConstraint(constraint); - // f.setFullTextConstraint(constraint); + ft.accept(new FullTextVisitor() { - return f; + @Override + public boolean visit(FullTextTerm term) { + return true; + } + + @Override + public boolean visit(FullTextAnd and) { + composite.set(true); + return true; + } + + @Override + public boolean visit(FullTextOr or) { + composite.set(true); + return true; + } + }); + return composite.get() && !hasNegativeContains(ft); } - static FullTextExpression getFlatConstraint( - FullTextExpression constraint) { + private static boolean hasNegativeContains(FullTextExpression ft) { + if (ft == null) { + return false; + } + final AtomicReference hasNegative = new AtomicReference(); + hasNegative.set(false); + + ft.accept(new FullTextVisitor.FullTextVisitorBase() { + + @Override + public boolean visit(FullTextTerm term) { + if (term.isNot()) { + hasNegative.set(true); + } + return true; + } + + }); + return hasNegative.get(); + } + + private static Cursor flatten(FullTextExpression constraint, + final Filter filter, final FulltextQueryIndex index, + final NodeState state) { if (constraint == null) { return null; } - final AtomicReference result = new AtomicReference(); + final AtomicReference result = new AtomicReference(); constraint.accept(new FullTextVisitor() { - + @Override public boolean visit(FullTextTerm term) { - String p = term.getPropertyName(); - if (p != null) { - if (PathUtils.getDepth(p) > 1) { - // remove indirection - String name = PathUtils.getName(p); - term = new FullTextTerm(name, term); - } + result.set(filterToCursor(newAggregationFilter(filter, term), + index, state)); + return true; + } + + @Override + public boolean visit(FullTextAnd and) { + Iterator iterator = and.list.iterator(); + Cursor c = flatten(iterator.next(), filter, index, state); + while (iterator.hasNext()) { + FullTextExpression input = iterator.next(); + Cursor newC = flatten(input, filter, index, state); + c = Cursors.newIntersectionCursor(c, newC, + filter.getQueryEngineSettings()); } - result.set(term); + result.set(c); + return true; + } + + @Override + public boolean visit(FullTextOr or) { + List cursors = Lists.transform(or.list, + new Function() { + @Override + public Cursor apply(FullTextExpression input) { + return flatten(input, filter, index, state); + } + }); + result.set(Cursors.newConcatCursor(cursors, + filter.getQueryEngineSettings())); + return true; + } + }); + return result.get(); + } + + private static double flattenCost(FullTextExpression constraint, + final Filter filter, final FulltextQueryIndex index, + final NodeState state) { + if (constraint == null) { + return Double.POSITIVE_INFINITY; + } + final AtomicReference result = new AtomicReference(); + result.set(0d); + constraint.accept(new FullTextVisitor() { + + @Override + public boolean visit(FullTextTerm term) { + result.set(result.get() + index.getCost(newAggregationFilter(filter, term), state)); return true; } @Override public boolean visit(FullTextAnd and) { - ArrayList list = new ArrayList(); - for (FullTextExpression e : and.list) { - list.add(getFlatConstraint(e)); + for (FullTextExpression input : and.list) { + double d = flattenCost(input, filter, index, state); + result.set(result.get() + d); } - result.set(new FullTextAnd(list)); return true; } @Override public boolean visit(FullTextOr or) { - ArrayList list = new ArrayList(); - for (FullTextExpression e : or.list) { - list.add(getFlatConstraint(e)); + for (FullTextExpression input : or.list) { + double d = flattenCost(input, filter, index, state); + result.set(result.get() + d); } - result.set(new FullTextOr(list)); return true; } - }); return result.get(); } + private static Cursor filterToCursor(Filter f, FulltextQueryIndex index, + NodeState state) { + return new AggregationCursor(index.query(f, state), + index.getNodeAggregator(), state); + } + + private static Filter newAggregationFilter(Filter filter, FullTextExpression exp) { + FilterImpl f = new FilterImpl(filter); + // disables node type checks for now + f.setMatchesAllTypes(true); + if (exp != null) { + f.setFullTextConstraint(exp); + } + return f; + } + @Override public String getPlan(Filter filter, NodeState rootState) { if (baseIndex == null) { Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java?rev=1583771&r1=1583770&r2=1583771&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java Tue Apr 1 20:10:17 2014 @@ -20,6 +20,7 @@ import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import javax.annotation.Nullable; @@ -57,6 +58,10 @@ public class Cursors { return new IntersectionCursor(a, b, settings); } + public static Cursor newConcatCursor(List cursors, QueryEngineSettings settings) { + return new ConcatCursor(cursors, settings); + } + /** * Creates a {@link Cursor} over paths. * @@ -361,9 +366,13 @@ public class Cursors { if (!init) { fetchNext(); init = true; + if (closed) { + throw new IllegalStateException("This cursor is closed"); + } } IndexRow result = current; - fetchNext(); + // fetchNext(); + init = false; return result; } @@ -402,8 +411,7 @@ public class Cursors { } secondSet.put(p2, s); FilterIterators.checkMemoryLimit(secondSet.size(), settings.getLimitInMemory()); - } - closed = true; + } } } @@ -413,5 +421,75 @@ public class Cursors { } } - + + /** + * A cursor that combines multiple cursors into a single cursor. + */ + private static class ConcatCursor extends AbstractCursor { + + private final HashSet seen = new HashSet(); + private final List cursors; + private final QueryEngineSettings settings; + private boolean init; + private boolean closed; + + private Cursor currentCursor; + private IndexRow current; + + ConcatCursor(List cursors, QueryEngineSettings settings) { + this.cursors = cursors; + this.settings = settings; + this.currentCursor = cursors.remove(0); + } + + @Override + public IndexRow next() { + if (closed) { + throw new IllegalStateException("This cursor is closed"); + } + if (!init) { + fetchNext(); + init = true; + } + IndexRow result = current; + fetchNext(); + return result; + } + + @Override + public boolean hasNext() { + if (!closed && !init) { + fetchNext(); + init = true; + } + return !closed; + } + + private void fetchNext() { + while (true) { + while (!currentCursor.hasNext()) { + if (cursors.isEmpty()) { + closed = true; + return; + } else { + currentCursor = cursors.remove(0); + } + } + IndexRow c = currentCursor.next(); + String p = c.getPath(); + if (seen.contains(p)) { + continue; + } + current = c; + markSeen(p); + return; + } + } + + private void markSeen(String path) { + seen.add(path); + FilterIterators.checkMemoryLimit(seen.size(), settings.getLimitInMemory()); + } + + } } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java Tue Apr 1 20:10:17 2014 @@ -45,7 +45,7 @@ public class CursorsTest { @Test public void intersectionCursorExceptions() { QueryEngineSettings s = new QueryEngineSettings(); - Cursor a = new SimpleCursor("1:", "/b", "/c", "/e", "/e", "/c"); + Cursor a = new SimpleCursor("1:", "/x", "/b", "/c", "/e", "/e", "/c"); Cursor b = new SimpleCursor("2:", "/a", "/c", "/d", "/b", "/c"); Cursor c = Cursors.newIntersectionCursor(a, b, s); c.next(); Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java Tue Apr 1 20:10:17 2014 @@ -34,11 +34,12 @@ import org.apache.jackrabbit.oak.api.Typ import org.apache.jackrabbit.oak.plugins.index.aggregate.AggregateIndexProvider; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator; + import static org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState.binaryProperty; + import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent; import org.apache.jackrabbit.oak.query.AbstractQueryTest; import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; -import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableList; @@ -361,7 +362,6 @@ public class LuceneIndexAggregationTest } @Test - @Ignore("OAK-828") public void testDifferentNodes() throws Exception { Tree folder = root.getTree("/").addChild("myFolder");