jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mreut...@apache.org
Subject svn commit: r219477 - in /incubator/jackrabbit/trunk/core: applications/test/ src/conf/ src/java/org/apache/jackrabbit/core/query/lucene/
Date Mon, 18 Jul 2005 12:20:46 GMT
Author: mreutegg
Date: Mon Jul 18 05:20:20 2005
New Revision: 219477

URL: http://svn.apache.org/viewcvs?rev=219477&view=rev
Log:
JCR-160: Query index not in sync with workspace
Extended consistency check to also include:
- Missing ancestor
- Unknown parent
- Nodes indexed more than once

Added:
    incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
  (with props)
    incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
  (with props)
Modified:
    incubator/jackrabbit/trunk/core/applications/test/repository.xml
    incubator/jackrabbit/trunk/core/src/conf/repository.xml
    incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
    incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java

Modified: incubator/jackrabbit/trunk/core/applications/test/repository.xml
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/applications/test/repository.xml?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/applications/test/repository.xml (original)
+++ incubator/jackrabbit/trunk/core/applications/test/repository.xml Mon Jul 18 05:20:20 2005
@@ -194,12 +194,19 @@
         <!--
             Search index and the file system it uses.
             class: FQN of class implementing the QueryHandler interface
-            Supported Parameters:
+
+            Supported parameters for lucene search index:
             - useCompoundFile: advises lucene to use compound files for the index files
             - minMergeDocs: minimum number of nodes in an index until segments are merged
             - maxMergeDocs: maximum number of nodes in segments that will be merged
             - mergeFactor: determines how often segment indices are merged
-            - bufferSize: maximum number of documents that are held in a pending queue until
added to the index
+            - bufferSize: maximum number of documents that are held in a pending
+              queue until added to the index
+            - forceConsistencyCheck: runs a consistency check on every startup. If
+              false, a consistency check is only performed when the search index
+              detects a prior forced shutdown.
+            - autoRepair: errors detected by a consistency check are automatically
+              repaired. If false, errors are only written to the log.
         -->
         <SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
             <param name="useCompoundFile" value="true"/>
@@ -207,7 +214,8 @@
             <param name="maxMergeDocs" value="100000"/>
             <param name="mergeFactor" value="10"/>
             <param name="bufferSize" value="10"/>
-
+            <param name="forceConsistencyCheck" value="false"/>
+            <param name="autoRepair" value="true"/>
             <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
                 <param name="path" value="${wsp.home}/index"/>
             </FileSystem>

Modified: incubator/jackrabbit/trunk/core/src/conf/repository.xml
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/conf/repository.xml?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/conf/repository.xml (original)
+++ incubator/jackrabbit/trunk/core/src/conf/repository.xml Mon Jul 18 05:20:20 2005
@@ -194,12 +194,19 @@
         <!--
             Search index and the file system it uses.
             class: FQN of class implementing the QueryHandler interface
-            Supported Parameters:
+
+            Supported parameters for lucene search index:
             - useCompoundFile: advises lucene to use compound files for the index files
             - minMergeDocs: minimum number of nodes in an index until segments are merged
             - maxMergeDocs: maximum number of nodes in segments that will be merged
             - mergeFactor: determines how often segment indices are merged
-            - bufferSize: maximum number of documents that are held in a pending queue until
added to the index
+            - bufferSize: maximum number of documents that are held in a pending
+              queue until added to the index
+            - forceConsistencyCheck: runs a consistency check on every startup. If
+              false, a consistency check is only performed when the search index
+              detects a prior forced shutdown.
+            - autoRepair: errors detected by a consistency check are automatically
+              repaired. If false, errors are only written to the log.
         -->
         <SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
             <param name="useCompoundFile" value="true"/>
@@ -207,7 +214,8 @@
             <param name="maxMergeDocs" value="100000"/>
             <param name="mergeFactor" value="10"/>
             <param name="bufferSize" value="10"/>
-
+            <param name="forceConsistencyCheck" value="false"/>
+            <param name="autoRepair" value="true"/>
             <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
                 <param name="path" value="${wsp.home}/index"/>
             </FileSystem>

Added: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java?rev=219477&view=auto
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
(added)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
Mon Jul 18 05:20:20 2005
@@ -0,0 +1,367 @@
+/*
+ * Copyright 2004-2005 The Apache Software Foundation or its licensors,
+ *                     as applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene;
+
+import org.apache.jackrabbit.core.state.ItemStateManager;
+import org.apache.jackrabbit.core.state.NodeState;
+import org.apache.jackrabbit.core.state.ItemStateException;
+import org.apache.jackrabbit.core.NodeId;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.document.Document;
+import org.apache.log4j.Logger;
+
+import javax.jcr.RepositoryException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
+
+/**
+ * Implements a consistency check on the search index. Currently the following
+ * checks are implemented:
+ * <ul>
+ * <li>Does not node exist in the ItemStateManager? If it does not exist
+ * anymore the node is deleted from the index.</li>
+ * <li>Is the parent of a node also present in the index? If it is not present it
+ * will be indexed.</li>
+ * <li>Is a node indexed multiple times? If that is the case, all occurrences
+ * in the index for such a node are removed, and the node is re-indexed.</li>
+ * </ul>
+ */
+class ConsistencyCheck {
+
+    /**
+     * Logger instance for this class
+     */
+    private static final Logger log = Logger.getLogger(ConsistencyCheck.class);
+
+    /**
+     * The ItemStateManager of the workspace.
+     */
+    private final ItemStateManager stateMgr;
+
+    /**
+     * The index to check.
+     */
+    private final MultiIndex index;
+
+    /**
+     * All the documents within the index.
+     */
+    private Map documents;
+
+    /**
+     * List of all errors.
+     */
+    private final List errors = new ArrayList();
+
+    /**
+     * Private constructor.
+     */
+    private ConsistencyCheck(MultiIndex index, ItemStateManager mgr) {
+        this.index = index;
+        this.stateMgr = mgr;
+    }
+
+    /**
+     * Runs the consistency check on <code>index</code>.
+     *
+     * @param index the index to check.
+     * @param mgr   the ItemStateManager from where to load content.
+     * @return the consistency check with the results.
+     * @throws IOException if an error occurs while checking.
+     */
+    static ConsistencyCheck run(MultiIndex index, ItemStateManager mgr) throws IOException
{
+        ConsistencyCheck check = new ConsistencyCheck(index, mgr);
+        check.run();
+        return check;
+    }
+
+    /**
+     * Repairs detected errors during the consistency check.
+     * @param ignoreFailure if <code>true</code> repair failures are ignored
+     *   the the repair continues without throwing an exception. If
+     *   <code>false</code> the repair procedure is aborted on the first
+     *   repair failure.
+     * @throws IOException if a repair failure occurs.
+     */
+    void repair(boolean ignoreFailure) throws IOException {
+        if (errors.size() == 0) {
+            log.info("No errors found.");
+            return;
+        }
+        int notRepairable = 0;
+        for (Iterator it = errors.iterator(); it.hasNext(); ) {
+            ConsistencyCheckError error = (ConsistencyCheckError) it.next();
+            try {
+                if (error.repairable()) {
+                    error.repair();
+                } else {
+                    notRepairable++;
+                }
+            } catch (IOException e) {
+                if (ignoreFailure) {
+                    log.warn("Exception while reparing: " + e);
+                } else {
+                    throw e;
+                }
+            }
+        }
+        log.info("Repaired " + (errors.size() - notRepairable) + " errors.");
+        if (notRepairable > 0) {
+            log.warn("" + notRepairable + " error(s) not repairable.");
+        }
+    }
+
+    /**
+     * Returns the errors detected by the consistency check.
+     * @return the errors detected by the consistency check.
+     */
+    List getErrors() {
+        return new ArrayList(errors);
+    }
+
+    /**
+     * Runs the consistency check.
+     * @throws IOException if an error occurs while running the check.
+     */
+    private void run() throws IOException {
+        // UUIDs of multiple nodes in the index
+        Set multipleEntries = new HashSet();
+        // collect all documents
+        documents = new HashMap();
+        IndexReader reader = index.getIndexReader();
+        for (int i = 0; i < reader.maxDoc(); i++) {
+            if (reader.isDeleted(i)) {
+                continue;
+            }
+            Document d = reader.document(i);
+            String uuid = d.get(FieldNames.UUID);
+            if (stateMgr.hasItemState(new NodeId(uuid))) {
+                Document old = (Document) documents.put(uuid, d);
+                if (old != null) {
+                    multipleEntries.add(uuid);
+                }
+            } else {
+                errors.add(new NodeDeleted(uuid));
+            }
+        }
+
+        // create multiple entries errors
+        for (Iterator it = multipleEntries.iterator(); it.hasNext(); ) {
+            errors.add(new MultipleEntries((String) it.next()));
+        }
+
+        // run through documents
+        for (Iterator it = documents.values().iterator(); it.hasNext(); ) {
+            Document d = (Document) it.next();
+            String uuid = d.get(FieldNames.UUID);
+            String parentUUID = d.get(FieldNames.PARENT);
+            if (documents.containsKey(parentUUID) || parentUUID.length() == 0) {
+                continue;
+            }
+            // parent is missing
+            NodeId parentId = new NodeId(parentUUID);
+            if (stateMgr.hasItemState(parentId)) {
+                errors.add(new MissingAncestor(uuid, parentUUID));
+            } else {
+                errors.add(new UnknownParent(uuid, parentUUID));
+            }
+        }
+    }
+
+    /**
+     * Returns the path for <code>node</code>. If an error occurs this method
+     * returns the uuid of the node.
+     *
+     * @param node the node to retrieve the path from
+     * @return the path of the node or its uuid.
+     */
+    private String getPath(NodeState node) {
+        // remember as fallback
+        String uuid = node.getUUID();
+        StringBuffer path = new StringBuffer();
+        List elements = new ArrayList();
+        try {
+            while (node.getParentUUID() != null) {
+                NodeId parentId = new NodeId(node.getParentUUID());
+                NodeState parent = (NodeState) stateMgr.getItemState(parentId);
+                NodeState.ChildNodeEntry entry = parent.getChildNodeEntry(node.getUUID());
+                elements.add(entry);
+                node = parent;
+            }
+            for (int i = elements.size() - 1; i > -1; i--) {
+                NodeState.ChildNodeEntry entry = (NodeState.ChildNodeEntry) elements.get(i);
+                path.append('/').append(entry.getName().getLocalName());
+                if (entry.getIndex() > 1) {
+                    path.append('[').append(entry.getIndex()).append(']');
+                }
+            }
+            if (path.length() == 0) {
+                path.append('/');
+            }
+            return path.toString();
+        } catch (ItemStateException e) {
+            return uuid;
+        }
+    }
+
+    //-------------------< ConsistencyCheckError classes >----------------------
+
+    /**
+     * One or more ancestors of an indexed node are not available in the index.
+     */
+    private class MissingAncestor extends ConsistencyCheckError {
+
+        private final String parentUUID;
+
+        private MissingAncestor(String uuid, String parentUUID) {
+            super("Parent of " + uuid + " missing in index. Parent: " + parentUUID, uuid);
+            this.parentUUID = parentUUID;
+        }
+
+        /**
+         * Returns <code>true</code>.
+         * @return <code>true</code>.
+         */
+        public boolean repairable() {
+            return true;
+        }
+
+        /**
+         * Repairs the missing node by indexing the missing ancestors.
+         * @throws IOException if an error occurs while repairing.
+         */
+        public void repair() throws IOException {
+            String pUUID = parentUUID;
+            while (!documents.containsKey(pUUID)) {
+                try {
+                    NodeState n = (NodeState) stateMgr.getItemState(new NodeId(pUUID));
+                    log.info("Reparing missing node " + getPath(n));
+                    Document d = index.createDocument(n);
+                    index.addDocument(d);
+                    documents.put(n.getUUID(), d);
+                    pUUID = n.getParentUUID();
+                } catch (ItemStateException e) {
+                    throw new IOException(e.toString());
+                } catch (RepositoryException e) {
+                    throw new IOException(e.toString());
+                }
+            }
+        }
+    }
+
+    /**
+     * The parent of a node is not available through the ItemStateManager.
+     */
+    private class UnknownParent extends ConsistencyCheckError {
+
+        private UnknownParent(String uuid, String parentUUID) {
+            super("Node " + uuid + " has unknown parent: " + parentUUID, uuid);
+        }
+
+        /**
+         * Not reparable (yet).
+         * @return <code>false</code>.
+         */
+        public boolean repairable() {
+            return false;
+        }
+
+        /**
+         * No operation.
+         */
+        public void repair() throws IOException {
+            log.warn("Unknown parent for " + uuid + " cannot be repaired");
+        }
+    }
+
+    /**
+     * A node is present multiple times in the index.
+     */
+    private class MultipleEntries extends ConsistencyCheckError {
+
+        MultipleEntries(String uuid) {
+            super("Multiple entries found for node " + uuid, uuid);
+        }
+
+        /**
+         * Returns <code>true</code>.
+         * @return <code>true</code>.
+         */
+        public boolean repairable() {
+            return true;
+        }
+
+        /**
+         * Removes the nodes with the identical uuids from the index and
+         * re-index the node.
+         * @throws IOException if an error occurs while repairing.
+         */
+        public void repair() throws IOException {
+            // first remove all occurrences
+            Term id = new Term(FieldNames.UUID, uuid);
+            while (index.removeDocument(id) > 0) {
+                // removed
+            }
+            // then re-index the node
+            try {
+                NodeState node = (NodeState) stateMgr.getItemState(new NodeId(uuid));
+                log.info("Re-indexing duplicate node occurrences in index: " + getPath(node));
+                Document d = index.createDocument(node);
+                index.addDocument(d);
+                documents.put(node.getUUID(), d);
+            } catch (ItemStateException e) {
+                throw new IOException(e.toString());
+            } catch (RepositoryException e) {
+                throw new IOException(e.toString());
+            }
+        }
+    }
+
+    /**
+     * Indicates that a node has been deleted but is still in the index.
+     */
+    private class NodeDeleted extends ConsistencyCheckError {
+
+        NodeDeleted(String uuid) {
+            super("Node " + uuid + " does not longer exist.", uuid);
+        }
+
+        /**
+         * Returns <code>true</code>.
+         * @return <code>true</code>.
+         */
+        public boolean repairable() {
+            return true;
+        }
+
+        /**
+         * Deletes the nodes from the index.
+         * @throws IOException if an error occurs while repairing.
+         */
+        public void repair() throws IOException {
+            log.info("Removing deleted node from index: " + uuid);
+            index.removeDocument(new Term(FieldNames.UUID, uuid));
+        }
+    }
+}

Propchange: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheck.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java?rev=219477&view=auto
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
(added)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
Mon Jul 18 05:20:20 2005
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2004-2005 The Apache Software Foundation or its licensors,
+ *                     as applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene;
+
+import java.io.IOException;
+
+/**
+ * Common base class for errors detected during the consistency check.
+ */
+abstract class ConsistencyCheckError {
+
+    /**
+     * Diagnostic message for this error.
+     */
+    protected final String message;
+
+    /**
+     * The UUID of the affected node.
+     */
+    protected final String uuid;
+
+    ConsistencyCheckError(String message, String uuid) {
+        this.message = message;
+        this.uuid = uuid;
+    }
+
+    /**
+     * Returns the diagnostic message.
+     * @return the diagnostic message.
+     */
+    public String toString() {
+        return message;
+    }
+
+    /**
+     * Returns <code>true</code> if this error can be repaired.
+     * @return <code>true</code> if this error can be repaired.
+     */
+    abstract boolean repairable();
+
+    /**
+     * Executes the repair operation.
+     * @throws IOException if an error occurs while repairing.
+     */
+    abstract void repair() throws IOException;
+}

Propchange: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/ConsistencyCheckError.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
(original)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java
Mon Jul 18 05:20:20 2005
@@ -42,7 +42,7 @@
  * A <code>MultiIndex</code> consists of a {@link VolatileIndex} and multiple
  * {@link PersistentIndex}es. The goal is to keep most parts of the index open
  * with index readers and write new index data to the volatile index. When
- * the volatile index reaches a certain size (see {@link #setMinMergeDocs(int)} a
+ * the volatile index reaches a certain size (see {@link SearchIndex#setMinMergeDocs(int)}
a
  * new persistent index is created with the index data from the volatile index.
  * the new persistent index is then added to the list of already existing
  * persistent indexes. Furhter operations on the new persistent index will
@@ -50,8 +50,8 @@
  * but also for delete operations on the index.
  * <p/>
  * The persistent indexes are merged from time to time. The merge behaviour
- * is configurable using the methods: {@link #setMaxMergeDocs(int)},
- * {@link #setMergeFactor(int)} and {@link #setMinMergeDocs(int)}. For detailed
+ * is configurable using the methods: {@link SearchIndex#setMaxMergeDocs(int)},
+ * {@link SearchIndex#setMergeFactor(int)} and {@link SearchIndex#setMinMergeDocs(int)}.
For detailed
  * description of the configuration parameters see also the lucene
  * <code>IndexWriter</code> class.
  * <p/>
@@ -85,37 +85,6 @@
     private static final String REDO_LOG = "redo.log";
 
     /**
-     * Compound file flag
-     */
-    private boolean useCompoundFile = true;
-
-    /**
-     * minMergeDocs config parameter
-     */
-    private int minMergeDocs = 1000;
-
-    /**
-     * maxMergeDocs config parameter
-     */
-    private int maxMergeDocs = 100000;
-
-    /**
-     * mergeFactor config parameter
-     */
-    private int mergeFactor = 10;
-
-    /**
-     * Default merge size: 1000
-     */
-    private static final long DEFAULT_MERGE_SIZE = 1000;
-
-    /**
-     * The maximum number of entries in the redo log until the volatile index
-     * is merged into the persistent one.
-     */
-    private long mergeSize = DEFAULT_MERGE_SIZE;
-
-    /**
      * Names of active persistent index directories.
      */
     private final IndexInfos indexNames = new IndexInfos();
@@ -152,6 +121,11 @@
     private IndexReader multiReader;
 
     /**
+     * <code>true</code> if the redo log contained entries on startup.
+     */
+    private boolean redoLogApplied = false;
+
+    /**
      * Creates a new MultiIndex.
      *
      * @param fs the base file system
@@ -186,10 +160,10 @@
                 FileSystem sub = new BasedFileSystem(fs, indexNames.getName(i));
                 sub.init();
                 PersistentIndex index = new PersistentIndex(indexNames.getName(i), sub, false,
handler.getAnalyzer());
-                index.setMaxMergeDocs(maxMergeDocs);
-                index.setMergeFactor(mergeFactor);
-                index.setMinMergeDocs(minMergeDocs);
-                index.setUseCompoundFile(useCompoundFile);
+                index.setMaxMergeDocs(handler.getMaxMergeDocs());
+                index.setMergeFactor(handler.getMergeFactor());
+                index.setMinMergeDocs(handler.getMinMergeDocs());
+                index.setUseCompoundFile(handler.getUseCompoundFile());
                 indexes.add(index);
             }
 
@@ -198,6 +172,9 @@
             RedoLog redoLog = new RedoLog(new FileSystemResource(fs, REDO_LOG));
 
             if (redoLog.hasEntries()) {
+                // when we have entries in the redo log there is no need to reindex
+                doInitialIndex = false;
+
                 log.warn("Found uncommitted redo log. Applying changes now...");
                 // apply changes to persistent index
                 Iterator it = redoLog.getEntries().iterator();
@@ -217,19 +194,12 @@
                 maybeMergeIndexes();
                 log.warn("Redo changes applied.");
                 redoLog.clear();
-            }
-
-            // execute integrity check on persistent indexes with locks on startup
-            for (Iterator it = indexes.iterator(); it.hasNext(); ) {
-                PersistentIndex index = (PersistentIndex) it.next();
-                if (index.getLockEncountered()) {
-                    log.info("Running integrity check on index: " + index.getName());
-                    index.integrityCheck(stateMgr);
-                }
+                redoLogApplied = true;
             }
 
             volatileIndex = new VolatileIndex(handler.getAnalyzer(), redoLog);
             volatileIndex.setUseCompoundFile(false);
+            volatileIndex.setBufferSize(handler.getBufferSize());
 
             if (doInitialIndex) {
                 // index root node
@@ -255,7 +225,7 @@
     synchronized void addDocument(Document doc) throws IOException {
         multiReader = null;
         volatileIndex.addDocument(doc);
-        if (volatileIndex.getRedoLog().getSize() >= mergeSize) {
+        if (volatileIndex.getRedoLog().getSize() >= handler.getMinMergeDocs()) {
             log.info("Committing in-memory index");
             commit();
         }
@@ -335,54 +305,24 @@
         return nsMappings;
     }
 
-    //-----------------------< properties >-------------------------------------
-
-    /**
-     * The lucene index writer property: useCompoundFile
-     */
-    void setUseCompoundFile(boolean b) {
-        useCompoundFile = b;
-        // apply to all persistent indexes
-        for (int i = 0; i < indexes.size(); i++) {
-            ((PersistentIndex) indexes.get(i)).setUseCompoundFile(b);
-        }
-    }
-
-    /**
-     * The lucene index writer property: minMergeDocs
-     */
-    void setMinMergeDocs(int minMergeDocs) {
-        this.minMergeDocs = minMergeDocs;
-        // apply to all persistent indexes
-        for (int i = 0; i < indexes.size(); i++) {
-            ((PersistentIndex) indexes.get(i)).setMinMergeDocs(minMergeDocs);
-        }
-    }
-
     /**
-     * The lucene index writer property: maxMergeDocs
+     * Returns a lucene Document for the <code>node</code>.
+     * @param node the node to index.
+     * @return the index document.
+     * @throws RepositoryException if an error occurs while reading from the
+     *   workspace.
      */
-    void setMaxMergeDocs(int maxMergeDocs) {
-        this.maxMergeDocs = maxMergeDocs;
-        // apply to all persistent indexes
-        for (int i = 0; i < indexes.size(); i++) {
-            ((PersistentIndex) indexes.get(i)).setMaxMergeDocs(maxMergeDocs);
-        }
+    Document createDocument(NodeState node) throws RepositoryException {
+        return handler.createDocument(node, nsMappings);
     }
 
     /**
-     * The lucene index writer property: mergeFactor
+     * Returns <code>true</code> if the redo log contained entries while
+     * this index was instantiated; <code>false</code> otherwise.
+     * @return <code>true</code> if the redo log contained entries.
      */
-    void setMergeFactor(int mergeFactor) {
-        this.mergeFactor = mergeFactor;
-        // apply to all persistent indexes
-        for (int i = 0; i < indexes.size(); i++) {
-            ((PersistentIndex) indexes.get(i)).setMergeFactor(mergeFactor);
-        }
-    }
-
-    public void setBufferSize(int size) {
-        volatileIndex.setBufferSize(size);
+    boolean getRedoLogApplied() {
+        return redoLogApplied;
     }
 
     //-------------------------< internal >-------------------------------------
@@ -401,10 +341,10 @@
         try {
             sub.init();
             index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
-            index.setMaxMergeDocs(maxMergeDocs);
-            index.setMergeFactor(mergeFactor);
-            index.setMinMergeDocs(minMergeDocs);
-            index.setUseCompoundFile(useCompoundFile);
+            index.setMaxMergeDocs(handler.getMaxMergeDocs());
+            index.setMergeFactor(handler.getMergeFactor());
+            index.setMinMergeDocs(handler.getMinMergeDocs());
+            index.setUseCompoundFile(handler.getUseCompoundFile());
             indexes.add(index);
             indexNames.addName(name);
             indexNames.write(fs);
@@ -427,6 +367,7 @@
         // create new volatile index
         volatileIndex = new VolatileIndex(handler.getAnalyzer(), volatileIndex.getRedoLog());
         volatileIndex.setUseCompoundFile(false);
+        volatileIndex.setBufferSize(handler.getBufferSize());
 
         maybeMergeIndexes();
     }
@@ -444,7 +385,7 @@
      */
     private void createIndex(NodeState node, ItemStateManager stateMgr)
             throws IOException, ItemStateException, RepositoryException {
-        addDocument(handler.createDocument(node, nsMappings));
+        addDocument(createDocument(node));
         List children = node.getChildNodeEntries();
         for (Iterator it = children.iterator(); it.hasNext();) {
             NodeState.ChildNodeEntry child = (NodeState.ChildNodeEntry) it.next();
@@ -468,7 +409,7 @@
             throws IOException {
         Document doc;
         try {
-            doc = handler.createDocument(node, nsMappings);
+            doc = createDocument(node);
         } catch (RepositoryException e) {
             log.warn("RepositoryException: " + e.getMessage());
             return;
@@ -480,10 +421,10 @@
                 FileSystem sub = new BasedFileSystem(fs, name);
                 sub.init();
                 PersistentIndex index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
-                index.setMaxMergeDocs(maxMergeDocs);
-                index.setMergeFactor(mergeFactor);
-                index.setMinMergeDocs(minMergeDocs);
-                index.setUseCompoundFile(useCompoundFile);
+                index.setMaxMergeDocs(handler.getMaxMergeDocs());
+                index.setMergeFactor(handler.getMergeFactor());
+                index.setMinMergeDocs(handler.getMinMergeDocs());
+                index.setUseCompoundFile(handler.getUseCompoundFile());
                 indexes.add(index);
                 indexNames.addName(name);
                 indexNames.write(fs);
@@ -518,8 +459,8 @@
 
     /**
      * Merges multiple persistent index into a single one according to the
-     * properties: {@link #setMaxMergeDocs(int)}, {@link #setMergeFactor(int)}
-     * and {@link #setMinMergeDocs(int)}.
+     * properties: {@link SearchIndex#setMaxMergeDocs(int)}, {@link
+     * SearchIndex#setMergeFactor(int)} and {@link SearchIndex#setMinMergeDocs(int)}.
      *
      * @throws IOException if an error occurs during the merge.
      */
@@ -546,9 +487,9 @@
         }
 
         // only check for merge if there are more than mergeFactor indexes
-        if (indexes.size() >= mergeFactor) {
-            long targetMergeDocs = minMergeDocs;
-            while (targetMergeDocs <= maxMergeDocs) {
+        if (indexes.size() >= handler.getMergeFactor()) {
+            long targetMergeDocs = handler.getMinMergeDocs();
+            while (targetMergeDocs <= handler.getMaxMergeDocs()) {
                 // find index smaller or equal than current target size
                 int minIndex = indexes.size();
                 int mergeDocs = 0;
@@ -561,14 +502,15 @@
                     mergeDocs += numDocs;
                 }
 
-                if (indexes.size() - (minIndex + 1) >= mergeFactor && mergeDocs
< maxMergeDocs) {
+                if (indexes.size() - (minIndex + 1) >= handler.getMergeFactor()
+                        && mergeDocs < handler.getMaxMergeDocs()) {
                     // found a merge to do
                     mergeIndex(minIndex + 1);
                 } else {
                     break;
                 }
                 // increase target size
-                targetMergeDocs *= mergeFactor;
+                targetMergeDocs *= handler.getMergeFactor();
             }
         }
     }
@@ -588,10 +530,10 @@
         try {
             sub.init();
             index = new PersistentIndex(name, sub, true, handler.getAnalyzer());
-            index.setMaxMergeDocs(maxMergeDocs);
-            index.setMergeFactor(mergeFactor);
-            index.setMinMergeDocs(minMergeDocs);
-            index.setUseCompoundFile(useCompoundFile);
+            index.setMaxMergeDocs(handler.getMaxMergeDocs());
+            index.setMergeFactor(handler.getMergeFactor());
+            index.setMinMergeDocs(handler.getMinMergeDocs());
+            index.setUseCompoundFile(handler.getUseCompoundFile());
         } catch (FileSystemException e) {
             throw new IOException(e.getMessage());
         }
@@ -627,7 +569,7 @@
     }
 
     /**
-     * <b>Note: This check will be removed when Jackrabbit 1.0 is final.</b>
+     * <b>todo: This check will be removed when Jackrabbit 1.0 is final.</b>
      * <p/>
      * Checks if an old index format is present and moves it to the new
      * subindex structure.

Modified: incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
URL: http://svn.apache.org/viewcvs/incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java?rev=219477&r1=219476&r2=219477&view=diff
==============================================================================
--- incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
(original)
+++ incubator/jackrabbit/trunk/core/src/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java
Mon Jul 18 05:20:20 2005
@@ -41,6 +41,8 @@
 import javax.jcr.RepositoryException;
 import javax.jcr.query.InvalidQueryException;
 import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
 
 /**
  * Implements a {@link org.apache.jackrabbit.core.query.QueryHandler} using
@@ -67,11 +69,53 @@
     private final FIFOReadWriteLock readWriteLock = new FIFOReadWriteLock();
 
     /**
+     * minMergeDocs config parameter.
+     */
+    private int minMergeDocs = 1000;
+
+    /**
+     * maxMergeDocs config parameter
+     */
+    private int maxMergeDocs = 100000;
+
+    /**
+     * mergeFactor config parameter
+     */
+    private int mergeFactor = 10;
+
+    /**
+     * Number of documents that are buffered before they are added to the index.
+     */
+    private int bufferSize = 10;
+
+    /**
+     * Compound file flag
+     */
+    private boolean useCompoundFile = true;
+
+    /**
      * Flag indicating whether document order is enable as the default ordering.
      */
     private boolean documentOrder = true;
 
     /**
+     * If set <code>true</code> the index is checked for consistency on startup.
+     * If <code>false</code> a consistency check is only performed when there
+     * are entries in the redo log on startup.
+     * <p/>
+     * Default value is: <code>false</code>.
+     */
+    private boolean forceConsistencyCheck = false;
+
+    /**
+     * If set <code>true</code> errors detected by the consistency check are
+     * repaired. If <code>false</code> the errors are only reported in the log.
+     * <p/>
+     * Default value is: <code>true</code>.
+     */
+    private boolean autoRepair = true;
+
+    /**
      * Default constructor.
      */
     public SearchIndex() {
@@ -87,6 +131,27 @@
             QueryHandlerContext context = getContext();
             index = new MultiIndex(context.getFileSystem(), this,
                     context.getItemStateManager(), context.getRootUUID());
+            if (index.getRedoLogApplied() || forceConsistencyCheck) {
+                log.info("Running consistency check...");
+                try {
+                    ConsistencyCheck check = ConsistencyCheck.run(index,
+                            context.getItemStateManager());
+                    if (autoRepair) {
+                        check.repair(true);
+                    } else {
+                        List errors = check.getErrors();
+                        if (errors.size() == 0) {
+                            log.info("No errors detected.");
+                        }
+                        for (Iterator it = errors.iterator(); it.hasNext(); ) {
+                            ConsistencyCheckError err = (ConsistencyCheckError) it.next();
+                            log.info(err.toString());
+                        }
+                    }
+                } catch (IOException e) {
+                    log.warn("Failed to run consistency check on index: " + e);
+                }
+            }
         } catch (FileSystemException e) {
             throw new IOException(e.getMessage());
         }
@@ -257,34 +322,107 @@
 
     //--------------------------< properties >----------------------------------
 
+    /**
+     * The lucene index writer property: useCompoundFile
+     */
     public void setUseCompoundFile(boolean b) {
-        index.setUseCompoundFile(b);
+        useCompoundFile = b;
+    }
+
+    /**
+     * Returns the current value for useCompoundFile.
+     *
+     * @return the current value for useCompoundFile.
+     */
+    public boolean getUseCompoundFile() {
+        return useCompoundFile;
     }
 
+    /**
+     * The lucene index writer property: minMergeDocs
+     */
     public void setMinMergeDocs(int minMergeDocs) {
-        index.setMinMergeDocs(minMergeDocs);
+        this.minMergeDocs = minMergeDocs;
     }
 
+    /**
+     * Returns the current value for minMergeDocs.
+     *
+     * @return the current value for minMergeDocs.
+     */
+    public int getMinMergeDocs() {
+        return minMergeDocs;
+    }
+
+    /**
+     * The lucene index writer property: maxMergeDocs
+     */
     public void setMaxMergeDocs(int maxMergeDocs) {
-        index.setMaxMergeDocs(maxMergeDocs);
+        this.maxMergeDocs = maxMergeDocs;
+    }
+
+    /**
+     * Returns the current value for maxMergeDocs.
+     *
+     * @return the current value for maxMergeDocs.
+     */
+    public int getMaxMergeDocs() {
+        return maxMergeDocs;
     }
 
+    /**
+     * The lucene index writer property: mergeFactor
+     */
     public void setMergeFactor(int mergeFactor) {
-        index.setMergeFactor(mergeFactor);
+        this.mergeFactor = mergeFactor;
     }
 
     /**
-     * @deprecated redo size is equivalent to minMergeDocs. Use
-     * {@link #setMinMergeDocs(int)} to set the size of the redo log.
+     * Returns the current value for the merge factor.
+     *
+     * @return the current value for the merge factor.
      */
-    public void setRedoSize(int size) {
+    public int getMergeFactor() {
+        return mergeFactor;
     }
 
+    /**
+     * @see VolatileIndex#setBufferSize(int)
+     */
     public void setBufferSize(int size) {
-        index.setBufferSize(size);
+        bufferSize = size;
+    }
+
+    /**
+     * Returns the current value for the buffer size.
+     *
+     * @return the current value for the buffer size.
+     */
+    public int getBufferSize() {
+        return bufferSize;
     }
 
     public void setRespectDocumentOrder(boolean docOrder) {
         documentOrder = docOrder;
+    }
+
+    public boolean getRespectDocumentOrder() {
+        return documentOrder;
+    }
+
+    public void setForceConsistencyCheck(boolean b) {
+        forceConsistencyCheck = b;
+    }
+
+    public boolean getForceConsistencyCheck() {
+        return forceConsistencyCheck;
+    }
+
+    public void setAutoRepair(boolean b) {
+        autoRepair = b;
+    }
+
+    public boolean getAutoRepair() {
+        return autoRepair;
     }
 }



Mime
View raw message