jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alexparvule...@apache.org
Subject svn commit: r1583771 - in /jackrabbit/oak/trunk: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/ oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/ oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/ oak-lucen...
Date Tue, 01 Apr 2014 20:10:18 GMT
Author: alexparvulescu
Date: Tue Apr  1 20:10:17 2014
New Revision: 1583771

URL: http://svn.apache.org/r1583771
Log:
OAK-1654 Composite index aggregates

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
    jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
Tue Apr  1 20:10:17 2014
@@ -16,15 +16,15 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.aggregate;
 
-import java.util.ArrayList;
+
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.jackrabbit.oak.api.PropertyValue;
-import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
 import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
 import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
@@ -32,14 +32,17 @@ import org.apache.jackrabbit.oak.query.f
 import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
 import org.apache.jackrabbit.oak.query.index.FilterImpl;
 import org.apache.jackrabbit.oak.spi.query.Cursor;
+import org.apache.jackrabbit.oak.spi.query.Cursors;
 import org.apache.jackrabbit.oak.spi.query.Cursors.AbstractCursor;
 import org.apache.jackrabbit.oak.spi.query.Filter;
 import org.apache.jackrabbit.oak.spi.query.IndexRow;
 import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
 
+import com.google.common.base.Function;
 import com.google.common.base.Predicates;
 import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
 
 /**
  * A virtual full-text that can aggregate nodes based on aggregate definitions.
@@ -58,7 +61,13 @@ public class AggregateIndex implements F
         if (baseIndex == null) {
             return Double.POSITIVE_INFINITY;
         }
-        return baseIndex.getCost(filter, rootState) - 0.05;
+        double localCost = Double.POSITIVE_INFINITY;
+        FullTextExpression e = filter.getFullTextConstraint();
+        if (e != null && hasCompositeExpression(e)) {
+            localCost = flattenCost(e, filter, baseIndex, rootState);
+        }
+        double baseCost = baseIndex.getCost(filter, rootState);
+        return Math.min(localCost, baseCost) - 0.05;
     }
 
     @Override
@@ -67,70 +76,172 @@ public class AggregateIndex implements F
         if (baseIndex.getNodeAggregator() == null) {
             return baseIndex.query(filter, rootState);
         }
-        return new AggregationCursor(baseIndex.query(
-                newAggregationFilter(filter), rootState),
-                baseIndex.getNodeAggregator(), rootState);
+        return newCursor(filter, baseIndex, rootState);
     }
 
-    private static Filter newAggregationFilter(Filter filter) {
-        FilterImpl f = new FilterImpl(filter);
-        // disables node type checks for now
-        f.setMatchesAllTypes(true);
+    private static Cursor newCursor(Filter f, FulltextQueryIndex index,
+            NodeState state) {
+        FullTextExpression e = f.getFullTextConstraint();
+        if (hasCompositeExpression(e)) {
+            Cursor c = flatten(e, f, index, state);
+            if (c != null) {
+                return c;
+            }
+        }
+        return new AggregationCursor(index.query(newAggregationFilter(f, null),
+                state), index.getNodeAggregator(), state);
+    }
+
+    private static boolean hasCompositeExpression(FullTextExpression ft) {
+        if (ft == null) {
+            return false;
+        }
+        final AtomicReference<Boolean> composite = new AtomicReference<Boolean>();
+        composite.set(false);
 
-        // TODO OAK-828
-        // FullTextExpression constraint = filter.getFullTextConstraint();
-        // constraint = getFlatConstraint(constraint);
-        // f.setFullTextConstraint(constraint);
+        ft.accept(new FullTextVisitor() {
 
-        return f;
+            @Override
+            public boolean visit(FullTextTerm term) {
+                return true;
+            }
+
+            @Override
+            public boolean visit(FullTextAnd and) {
+                composite.set(true);
+                return true;
+            }
+
+            @Override
+            public boolean visit(FullTextOr or) {
+                composite.set(true);
+                return true;
+            }
+        });
+        return composite.get() && !hasNegativeContains(ft);
     }
 
-    static FullTextExpression getFlatConstraint(
-            FullTextExpression constraint) {
+    private static boolean hasNegativeContains(FullTextExpression ft) {
+        if (ft == null) {
+            return false;
+        }
+        final AtomicReference<Boolean> hasNegative = new AtomicReference<Boolean>();
+        hasNegative.set(false);
+
+        ft.accept(new FullTextVisitor.FullTextVisitorBase() {
+
+            @Override
+            public boolean visit(FullTextTerm term) {
+                if (term.isNot()) {
+                    hasNegative.set(true);
+                }
+                return true;
+            }
+
+        });
+        return hasNegative.get();
+    }
+
+    private static Cursor flatten(FullTextExpression constraint,
+            final Filter filter, final FulltextQueryIndex index,
+            final NodeState state) {
         if (constraint == null) {
             return null;
         }
-        final AtomicReference<FullTextExpression> result = new AtomicReference<FullTextExpression>();
+        final AtomicReference<Cursor> result = new AtomicReference<Cursor>();
         constraint.accept(new FullTextVisitor() {
-            
+
             @Override
             public boolean visit(FullTextTerm term) {
-                String p = term.getPropertyName();
-                if (p != null) {
-                    if (PathUtils.getDepth(p) > 1) {
-                        // remove indirection
-                        String name = PathUtils.getName(p);
-                        term = new FullTextTerm(name, term);
-                    }
+                result.set(filterToCursor(newAggregationFilter(filter, term),
+                        index, state));
+                return true;
+            }
+
+            @Override
+            public boolean visit(FullTextAnd and) {
+                Iterator<FullTextExpression> iterator = and.list.iterator();
+                Cursor c = flatten(iterator.next(), filter, index, state);
+                while (iterator.hasNext()) {
+                    FullTextExpression input = iterator.next();
+                    Cursor newC = flatten(input, filter, index, state);
+                    c = Cursors.newIntersectionCursor(c, newC,
+                            filter.getQueryEngineSettings());
                 }
-                result.set(term);
+                result.set(c);
+                return true;
+            }
+
+            @Override
+            public boolean visit(FullTextOr or) {
+                List<Cursor> cursors = Lists.transform(or.list,
+                        new Function<FullTextExpression, Cursor>() {
+                            @Override
+                            public Cursor apply(FullTextExpression input) {
+                                return flatten(input, filter, index, state);
+                            }
+                        });
+                result.set(Cursors.newConcatCursor(cursors,
+                        filter.getQueryEngineSettings()));
+                return true;
+            }
+        });
+        return result.get();
+    }
+
+    private static double flattenCost(FullTextExpression constraint,
+            final Filter filter, final FulltextQueryIndex index,
+            final NodeState state) {
+        if (constraint == null) {
+            return Double.POSITIVE_INFINITY;
+        }
+        final AtomicReference<Double> result = new AtomicReference<Double>();
+        result.set(0d);
+        constraint.accept(new FullTextVisitor() {
+
+            @Override
+            public boolean visit(FullTextTerm term) {
+                result.set(result.get() + index.getCost(newAggregationFilter(filter, term),
state));
                 return true;
             }
 
             @Override
             public boolean visit(FullTextAnd and) {
-                ArrayList<FullTextExpression> list = new ArrayList<FullTextExpression>();
-                for (FullTextExpression e : and.list) {
-                    list.add(getFlatConstraint(e));
+                for (FullTextExpression input : and.list) {
+                    double d = flattenCost(input, filter, index, state);
+                    result.set(result.get() + d);
                 }
-                result.set(new FullTextAnd(list));
                 return true;
             }
 
             @Override
             public boolean visit(FullTextOr or) {
-                ArrayList<FullTextExpression> list = new ArrayList<FullTextExpression>();
-                for (FullTextExpression e : or.list) {
-                    list.add(getFlatConstraint(e));
+                for (FullTextExpression input : or.list) {
+                    double d = flattenCost(input, filter, index, state);
+                    result.set(result.get() + d);
                 }
-                result.set(new FullTextOr(list));
                 return true;
             }
-            
         });
         return result.get();
     }
 
+    private static Cursor filterToCursor(Filter f, FulltextQueryIndex index,
+            NodeState state) {
+        return new AggregationCursor(index.query(f, state),
+                index.getNodeAggregator(), state);
+    }
+
+    private static Filter newAggregationFilter(Filter filter, FullTextExpression exp) {
+        FilterImpl f = new FilterImpl(filter);
+        // disables node type checks for now
+        f.setMatchesAllTypes(true);
+        if (exp != null) {
+            f.setFullTextConstraint(exp);
+        }
+        return f;
+    }
+
     @Override
     public String getPlan(Filter filter, NodeState rootState) {
         if (baseIndex == null) {

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
Tue Apr  1 20:10:17 2014
@@ -20,6 +20,7 @@ import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 
 import javax.annotation.Nullable;
 
@@ -57,6 +58,10 @@ public class Cursors {
         return new IntersectionCursor(a, b, settings);
     }
 
+    public static Cursor newConcatCursor(List<Cursor> cursors, QueryEngineSettings
settings) {
+        return new ConcatCursor(cursors, settings);
+    }
+
     /**
      * Creates a {@link Cursor} over paths.
      *
@@ -361,9 +366,13 @@ public class Cursors {
             if (!init) {
                 fetchNext();
                 init = true;
+                if (closed) {
+                    throw new IllegalStateException("This cursor is closed");
+                }
             }
             IndexRow result = current;
-            fetchNext();
+            // fetchNext();
+            init = false;
             return result;
         }
 
@@ -402,8 +411,7 @@ public class Cursors {
                     }
                     secondSet.put(p2, s);
                     FilterIterators.checkMemoryLimit(secondSet.size(), settings.getLimitInMemory());
-                }                    
-                closed = true;
+                }
             }
         }
         
@@ -413,5 +421,75 @@ public class Cursors {
         }
         
     }
-    
+
+    /**
+     * A cursor that combines multiple cursors into a single cursor.
+     */
+    private static class ConcatCursor extends AbstractCursor {
+
+        private final HashSet<String> seen = new HashSet<String>();
+        private final List<Cursor> cursors;
+        private final QueryEngineSettings settings;
+        private boolean init;
+        private boolean closed;
+
+        private Cursor currentCursor;
+        private IndexRow current;
+
+        ConcatCursor(List<Cursor> cursors, QueryEngineSettings settings) {
+            this.cursors = cursors;
+            this.settings = settings;
+            this.currentCursor = cursors.remove(0);
+        }
+
+        @Override
+        public IndexRow next() {
+            if (closed) {
+                throw new IllegalStateException("This cursor is closed");
+            }
+            if (!init) {
+                fetchNext();
+                init = true;
+            }
+            IndexRow result = current;
+            fetchNext();
+            return result;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (!closed && !init) {
+                fetchNext();
+                init = true;
+            }
+            return !closed;
+        }
+
+        private void fetchNext() {
+            while (true) {
+                while (!currentCursor.hasNext()) {
+                    if (cursors.isEmpty()) {
+                        closed = true;
+                        return;
+                    } else {
+                        currentCursor = cursors.remove(0);
+                    }
+                }
+                IndexRow c = currentCursor.next();
+                String p = c.getPath();
+                if (seen.contains(p)) {
+                    continue;
+                }
+                current = c;
+                markSeen(p);
+                return;
+            }
+        }
+
+        private void markSeen(String path) {
+            seen.add(path);
+            FilterIterators.checkMemoryLimit(seen.size(), settings.getLimitInMemory());
+        }
+
+    }
 }

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
Tue Apr  1 20:10:17 2014
@@ -45,7 +45,7 @@ public class CursorsTest {
     @Test
     public void intersectionCursorExceptions() {
         QueryEngineSettings s = new QueryEngineSettings();
-        Cursor a = new SimpleCursor("1:", "/b", "/c", "/e", "/e", "/c");
+        Cursor a = new SimpleCursor("1:", "/x", "/b", "/c", "/e", "/e", "/c");
         Cursor b = new SimpleCursor("2:", "/a", "/c", "/d", "/b", "/c");
         Cursor c = Cursors.newIntersectionCursor(a, b, s);
         c.next();

Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
(original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
Tue Apr  1 20:10:17 2014
@@ -34,11 +34,12 @@ import org.apache.jackrabbit.oak.api.Typ
 import org.apache.jackrabbit.oak.plugins.index.aggregate.AggregateIndexProvider;
 import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
 import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator;
+
 import static org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState.binaryProperty;
+
 import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
 import org.apache.jackrabbit.oak.query.AbstractQueryTest;
 import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.ImmutableList;
@@ -361,7 +362,6 @@ public class LuceneIndexAggregationTest 
     }
 
     @Test
-     @Ignore("OAK-828")
     public void testDifferentNodes() throws Exception {
 
         Tree folder = root.getTree("/").addChild("myFolder");



Mime
View raw message