Mailing-List: contact oak-commits-help@jackrabbit.apache.org; run by ezmlm
Precedence: bulk
Reply-To: oak-dev@jackrabbit.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1389824 - in /jackrabbit/oak/trunk/oak-core/src:
 main/java/org/apache/jackrabbit/oak/plugins/lucene/
 test/java/org/apache/jackrabbit/oak/plugins/lucene/
Date: Tue, 25 Sep 2012 12:09:20 -0000
To: oak-commits@jackrabbit.apache.org
From: alexparvulescu@apache.org
Message-Id: <20120925120921.332F623888CD@eris.apache.org>

Author: alexparvulescu
Date: Tue Sep 25 12:09:20 2012
New Revision: 1389824

URL: http://svn.apache.org/viewvc?rev=1389824&view=rev
Log:
OAK-340 Basic reindex support in the lucene index
 - add LuceneReindexHook, a hook that triggers reindexing on any detected index definition change
 - tweaked the LuceneEditor to trigger a reindex from #processCommit of the before state is null.
 - played with the way paths are stored & queried in the index

Added:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java   (with props)
Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneEditor.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneIndex.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/lucene/AbstractLuceneQueryTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneEditor.java?rev=1389824&r1=1389823&r2=1389824&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneEditor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneEditor.java Tue Sep 25 12:09:20 2012
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.oak.plugins.lucene;
 
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
 import static org.apache.jackrabbit.oak.plugins.lucene.FieldFactory.newPathField;
 import static org.apache.jackrabbit.oak.plugins.lucene.FieldFactory.newPropertyField;
 import static org.apache.jackrabbit.oak.plugins.lucene.TermFactory.newPathTerm;
@@ -77,6 +78,11 @@ class LuceneEditor implements CommitHook
         this.path = split(indexDefinition.getPath(), INDEX_DATA_CHILD_NAME);
     }
 
+    /*
+     * 
+     * If before is null, then the #processCommit call is treated as a full
+     * reindex call
+     */
     @Override
     public NodeState processCommit(NodeStore store, NodeState before,
             NodeState after) throws CommitFailedException {
@@ -91,8 +97,16 @@ class LuceneEditor implements CommitHook
         try {
             IndexWriter writer = new IndexWriter(directory, config);
             try {
-                LuceneDiff diff = new LuceneDiff(writer, "");
-                after.compareAgainstBaseState(before, diff);
+                LuceneDiff diff = new LuceneDiff(writer, "/");
+                if (before != null) {
+                    // normal diff
+                    after.compareAgainstBaseState(before, diff);
+                } else {
+                    // trigger re-indexing
+                    diff.childNodeDeleted("", after);
+                    diff.childNodeAdded("", after);
+                }
+
                 diff.postProcess(after);
                 writer.commit();
             } finally {
@@ -153,13 +167,24 @@ class LuceneEditor implements CommitHook
             }
             if (exception == null) {
                 try {
-                    addSubtree(path + "/" + name, after);
+                    addSubtree(path + name, after);
                 } catch (IOException e) {
                     exception = e;
                 }
             }
         }
 
+        private void addSubtree(String path, NodeState state)
+                throws IOException {
+            writer.addDocument(makeDocument(path, state));
+            for (ChildNodeEntry entry : state.getChildNodeEntries()) {
+                if (NodeStateUtils.isHidden(entry.getName())) {
+                    continue;
+                }
+                addSubtree(concat(path, entry.getName()), entry.getNodeState());
+            }
+        }
+
         @Override
         public void childNodeChanged(String name, NodeState before,
                 NodeState after) {
@@ -168,7 +193,7 @@ class LuceneEditor implements CommitHook
             }
             if (exception == null) {
                 try {
-                    LuceneDiff diff = new LuceneDiff(writer, path + "/" + name);
+                    LuceneDiff diff = new LuceneDiff(writer, path + name);
                     after.compareAgainstBaseState(before, diff);
                     diff.postProcess(after);
                 } catch (IOException e) {
@@ -184,26 +209,21 @@ class LuceneEditor implements CommitHook
             }
             if (exception == null) {
                 try {
-                    deleteSubtree(path + "/" + name, before);
+                    deleteSubtree(path + name, before);
                 } catch (IOException e) {
                     exception = e;
                 }
             }
         }
 
-        private void addSubtree(String path, NodeState state)
-                throws IOException {
-            writer.addDocument(makeDocument(path, state));
-            for (ChildNodeEntry entry : state.getChildNodeEntries()) {
-                addSubtree(path + "/" + entry.getName(), entry.getNodeState());
-            }
-        }
-
         private void deleteSubtree(String path, NodeState state)
                 throws IOException {
             writer.deleteDocuments(newPathTerm(path));
             for (ChildNodeEntry entry : state.getChildNodeEntries()) {
-                deleteSubtree(path + "/" + entry.getName(),
+                if (NodeStateUtils.isHidden(entry.getName())) {
+                    continue;
+                }
+                deleteSubtree(concat(path, entry.getName()),
                         entry.getNodeState());
             }
         }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneIndex.java?rev=1389824&r1=1389823&r2=1389824&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneIndex.java Tue Sep 25 12:09:20 2012
@@ -28,6 +28,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.query.index.IndexRowImpl;
 import org.apache.jackrabbit.oak.spi.query.Cursor;
 import org.apache.jackrabbit.oak.spi.query.Filter;
@@ -137,29 +138,29 @@ public class LuceneIndex implements Quer
         List<Query> qs = new ArrayList<Query>();
 
         String path = filter.getPath();
-        if (path.equals("/")) {
-            path = "";
-        }
         switch (filter.getPathRestriction()) {
         case ALL_CHILDREN:
-            qs.add(new PrefixQuery(newPathTerm(path + "/")));
+            if (!path.endsWith("/")) {
+                path += "/";
+            }
+            qs.add(new PrefixQuery(newPathTerm(path)));
             break;
         case DIRECT_CHILDREN:
             // FIXME
-            qs.add(new PrefixQuery(newPathTerm(path + "/")));
+            if (!path.endsWith("/")) {
+                path += "/";
+            }
+            qs.add(new PrefixQuery(newPathTerm(path)));
             break;
         case EXACT:
             qs.add(new TermQuery(newPathTerm(path)));
             break;
         case PARENT:
-            int slash = path.lastIndexOf('/');
-            if (slash != -1) {
-                String parent = path.substring(0, slash);
-                qs.add(new TermQuery(newPathTerm(parent)));
-            } else {
+            if (PathUtils.denotesRoot(path)) {
                 // there's no parent of the root node
                 return null;
             }
+            qs.add(new TermQuery(newPathTerm(PathUtils.getParentPath(path))));
             break;
         }
 

Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java?rev=1389824&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java Tue Sep 25 12:09:20 2012
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.lucene;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.spi.commit.CommitHook;
+import org.apache.jackrabbit.oak.spi.commit.CompositeHook;
+import org.apache.jackrabbit.oak.spi.query.IndexDefinition;
+import org.apache.jackrabbit.oak.spi.query.IndexUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link CommitHook} that handles re-indexing and initial indexing of the
+ * content.
+ * 
+ * Currently it triggers a full reindex on any detected index definition change.
+ * 
+ */
+public class LuceneReindexHook implements CommitHook, LuceneIndexConstants {
+
+    private static final Logger LOG = LoggerFactory
+            .getLogger(LuceneReindexHook.class);
+
+    private final String indexConfigPath;
+
+    public LuceneReindexHook(String indexConfigPath) {
+        this.indexConfigPath = indexConfigPath;
+    }
+
+    /**
+     * TODO test only
+     */
+    public LuceneReindexHook() {
+        this(IndexUtils.DEFAULT_INDEX_HOME);
+    }
+
+    @Override
+    public NodeState processCommit(NodeStore store, NodeState before,
+            NodeState after) throws CommitFailedException {
+
+        List<IndexDefinition> defsBefore = IndexUtils.buildIndexDefinitions(
+                before, indexConfigPath, TYPE);
+        List<IndexDefinition> defsAfter = IndexUtils.buildIndexDefinitions(
+                after, indexConfigPath, TYPE);
+
+        // TODO add a more fine-grained change verifier
+        List<IndexDefinition> defsChanged = new ArrayList<IndexDefinition>();
+        for (IndexDefinition def : defsAfter) {
+            if (!defsBefore.contains(def)) {
+                defsChanged.add(def);
+            }
+        }
+        if (defsChanged.isEmpty()) {
+            return after;
+        }
+        LOG.debug("found {} updated index definitions ({})",
+                defsChanged.size(), defsChanged);
+        LOG.debug("reindexing repository content");
+        long t = System.currentTimeMillis();
+        // TODO buffer content reindex
+        List<CommitHook> hooks = new ArrayList<CommitHook>();
+        for (IndexDefinition def : defsChanged) {
+            hooks.add(new LuceneEditor(def));
+        }
+        if (hooks.isEmpty()) {
+            return after;
+        }
+        NodeState done = new CompositeHook(hooks).processCommit(store, null,
+                after);
+        LOG.debug("done reindexing repository content in {} ms.",
+                System.currentTimeMillis() - t);
+        return done;
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/lucene/LuceneReindexHook.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/lucene/AbstractLuceneQueryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/lucene/AbstractLuceneQueryTest.java?rev=1389824&r1=1389823&r2=1389824&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/lucene/AbstractLuceneQueryTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/lucene/AbstractLuceneQueryTest.java Tue Sep 25 12:09:20 2012
@@ -32,6 +32,8 @@ import org.apache.jackrabbit.oak.api.Tre
 import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.core.ContentRepositoryImpl;
 import org.apache.jackrabbit.oak.core.DefaultConflictHandler;
+import org.apache.jackrabbit.oak.spi.commit.CommitHook;
+import org.apache.jackrabbit.oak.spi.commit.CompositeHook;
 import org.apache.jackrabbit.oak.spi.query.CompositeQueryIndexProvider;
 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
 import org.junit.Before;
@@ -62,13 +64,14 @@ public abstract class AbstractLuceneQuer
 
     @Override
     protected ContentRepository createRepository() {
-        QueryIndexProvider indexer = new LuceneIndexProvider(DEFAULT_INDEX_HOME);
-        QueryIndexProvider qip = new CompositeQueryIndexProvider(indexer);
-        return new ContentRepositoryImpl(new MicroKernelImpl(), qip,
-                new LuceneHook(DEFAULT_INDEX_HOME));
+        QueryIndexProvider qip = new CompositeQueryIndexProvider(
+                new LuceneIndexProvider(DEFAULT_INDEX_HOME));
+        CommitHook ch = new CompositeHook(new LuceneHook(DEFAULT_INDEX_HOME),
+                new LuceneReindexHook(DEFAULT_INDEX_HOME));
+        return new ContentRepositoryImpl(new MicroKernelImpl(), qip, ch);
     }
 
-    private void createIndexNode() throws Exception {
+    protected void createIndexNode() throws Exception {
         Tree index = root.getTree("/");
         for (String p : PathUtils.elements(DEFAULT_INDEX_HOME)) {
             if (index.hasChild(p)) {