jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mreut...@apache.org
Subject svn commit: r587868 - in /jackrabbit/sandbox/jackrabbit-spellchecker: ./ src/ src/main/ src/main/java/ src/main/java/org/ src/main/java/org/apache/ src/main/java/org/apache/jackrabbit/ src/main/java/org/apache/jackrabbit/core/ src/main/java/org/apache/...
Date Wed, 24 Oct 2007 12:17:53 GMT
Author: mreutegg
Date: Wed Oct 24 05:17:51 2007
New Revision: 587868

URL: http://svn.apache.org/viewvc?rev=587868&view=rev
Log:
JCR-1184: Introduce spellchecker functionality based on content in the workspace

Added:
    jackrabbit/sandbox/jackrabbit-spellchecker/   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/src/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties   (with props)
    jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml   (with props)

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Oct 24 05:17:51 2007
@@ -0,0 +1,5 @@
+target
+*.iml
+*.ipr
+*.iws
+*.log

Added: jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt Wed Oct 24 05:17:51 2007
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/LICENSE.txt
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt Wed Oct 24 05:17:51 2007
@@ -0,0 +1,5 @@
+Apache Jackrabbit
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/NOTICE.txt
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml Wed Oct 24 05:17:51 2007
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.jackrabbit</groupId>
+    <artifactId>jackrabbit</artifactId>
+    <version>1.4-SNAPSHOT</version>
+  </parent>
+  <artifactId>jackrabbit-spellchecker</artifactId>
+  <packaging>jar</packaging>
+
+  <name>Jackrabbit spell checker</name>
+  <description>Jackrabbit spell checker</description>
+
+  <!-- Dependencies -->
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.jackrabbit</groupId>
+      <artifactId>jackrabbit-core</artifactId>
+      <version>1.4-SNAPSHOT</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-spellchecker</artifactId>
+      <version>2.2.0</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>javax.jcr</groupId>
+      <artifactId>jcr</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/pom.xml
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java Wed Oct 24 05:17:51 2007
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene.spell;
+
+import org.apache.jackrabbit.core.query.lucene.SearchIndex;
+import org.apache.jackrabbit.core.query.lucene.FieldNames;
+import org.apache.jackrabbit.core.query.QueryHandler;
+import org.apache.jackrabbit.core.query.QueryRootNode;
+import org.apache.jackrabbit.core.query.TraversingQueryNodeVisitor;
+import org.apache.jackrabbit.core.query.RelationQueryNode;
+import org.apache.lucene.search.spell.SpellChecker;
+import org.apache.lucene.search.spell.Dictionary;
+import org.apache.lucene.search.spell.LuceneDictionary;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.NativeFSLockFactory;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * <code>LuceneSpellChecker</code> implements a spell checker based on the terms
+ * present in a lucene index.
+ */
+public class LuceneSpellChecker
+        implements org.apache.jackrabbit.core.query.lucene.SpellChecker {
+
+    /**
+     * Logger instance for this class.
+     */
+    private static final Logger log = LoggerFactory.getLogger(LuceneSpellChecker.class);
+
+    public static final class FiveSecondsRefreshInterval extends LuceneSpellChecker {
+        public FiveSecondsRefreshInterval() {
+            super(5 * 1000);
+        }
+    }
+
+    public static final class OneMinuteRefreshInterval extends LuceneSpellChecker {
+        public OneMinuteRefreshInterval() {
+            super(60 * 1000);
+        }
+    }
+
+    public static final class FiveMinutesRefreshInterval extends LuceneSpellChecker {
+        public FiveMinutesRefreshInterval() {
+            super(5 * 60 * 1000);
+        }
+    }
+
+    public static final class ThirtyMinutesRefreshInterval extends LuceneSpellChecker {
+        public ThirtyMinutesRefreshInterval() {
+            super(30 * 60 * 1000);
+        }
+    }
+
+    public static final class OneHourRefreshInterval extends LuceneSpellChecker {
+        public OneHourRefreshInterval() {
+            super(60 * 60 * 1000);
+        }
+    }
+
+    public static final class SixHoursRefreshInterval extends LuceneSpellChecker {
+        public SixHoursRefreshInterval() {
+            super(6 * 60 * 60 * 1000);
+        }
+    }
+
+    public static final class TwelveHoursRefreshInterval extends LuceneSpellChecker {
+        public TwelveHoursRefreshInterval() {
+            super(12 * 60 * 60 * 1000);
+        }
+    }
+
+    public static final class OneDayRefreshInterval extends LuceneSpellChecker {
+        public OneDayRefreshInterval() {
+            super(24 * 60 * 60 * 1000);
+        }
+    }
+
+    /**
+     * The internal spell checker.
+     */
+    private InternalSpellChecker spellChecker;
+
+    /**
+     * The refresh interval.
+     */
+    private final long refreshInterval;
+
+    /**
+     * Spell checker with a default refresh interval of one hour.
+     */
+    public LuceneSpellChecker() {
+        this(60 * 60 * 1000); // default refresh interval: one hour
+    }
+
+    protected LuceneSpellChecker(long refreshInterval) {
+        this.refreshInterval = refreshInterval;
+    }
+
+    /**
+     * Initializes this spell checker.
+     *
+     * @param handler the query handler that created this spell checker.
+     * @throws IOException if <code>handler</code> is not of type {@link
+     *                     SearchIndex}.
+     */
+    public void init(QueryHandler handler)
+            throws IOException {
+        if (handler instanceof SearchIndex) {
+            this.spellChecker = new InternalSpellChecker((SearchIndex) handler);
+        } else {
+            throw new IOException("LuceneSpellChecker only works with " +
+                    SearchIndex.class.getName());
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public String check(QueryRootNode aqt) throws IOException {
+        String stmt = getFulltextStatement(aqt);
+        if (stmt == null) {
+            // no spellcheck operation in query
+            return null;
+        }
+        return spellChecker.suggest(stmt);
+    }
+
+    public void close() {
+        spellChecker.close();
+    }
+
+    //------------------------------< internal >--------------------------------
+
+    /**
+     * Returns the fulltext statement of a spellcheck relation query node or
+     * <code>null</code> if none exists in the abstract query tree.
+     *
+     * @param aqt the abstract query tree.
+     * @return the fulltext statement or <code>null</code>.
+     */
+    private String getFulltextStatement(QueryRootNode aqt) {
+        final String[] stmt = new String[1];
+        aqt.accept(new TraversingQueryNodeVisitor() {
+            public Object visit(RelationQueryNode node, Object o) {
+                if (stmt[0] == null && node.getOperation() == RelationQueryNode.OPERATION_SPELLCHECK) {
+                    stmt[0] = node.getStringValue();
+                }
+                return super.visit(node, o);
+            }
+        }, null);
+        return stmt[0];
+    }
+
+    private final class InternalSpellChecker {
+
+        /**
+         * Timestamp when the last refresh was done.
+         */
+        private long lastRefresh;
+
+        /**
+         * Set to true while a refresh is done in a separate thread.
+         */
+        private boolean refreshing = false;
+
+        /**
+         * The query handler associated with this spell checker.
+         */
+        private final SearchIndex handler;
+
+        /**
+         * The directory where the spell index is stored.
+         */
+        private final Directory spellIndexDirectory;
+
+        /**
+         * The underlying spell checker.
+         */
+        private SpellChecker spellChecker;
+
+        /**
+         * Creates a new internal spell checker.
+         * @param handler the associated query handler.
+         */
+        InternalSpellChecker(SearchIndex handler) throws IOException {
+            this.handler = handler;
+            String path = handler.getPath() + File.separatorChar + "spellchecker";
+            this.spellIndexDirectory = FSDirectory.getDirectory(
+                    path, new NativeFSLockFactory(path));
+            if (IndexReader.indexExists(spellIndexDirectory)) {
+                this.lastRefresh = System.currentTimeMillis();
+            }
+            this.spellChecker = new SpellChecker(spellIndexDirectory);
+            refreshSpellChecker();
+        }
+
+        /**
+         * Checks a fulltext query statement and suggests a spell checked
+         * version of the statement. If the spell checker thinks the spelling is
+         * correct <code>null</code> is returned.
+         *
+         * @param statement the fulltext query statement.
+         * @return a suggestion or <code>null</code>.
+         */
+        String suggest(String statement) throws IOException {
+            // tokenize the statement (field name doesn't matter actually...)
+            List words = new ArrayList();
+            List tokens = new ArrayList();
+            tokenize(statement, words, tokens);
+
+            String[] suggestions = check(
+                    (String[]) words.toArray(new String[words.size()]));
+            if (suggestions != null) {
+                // replace words in statement in reverse order because length
+                // of statement will change
+                StringBuffer sb = new StringBuffer(statement);
+                for (int i = suggestions.length - 1; i >= 0; i--) {
+                    Token t = (Token) tokens.get(i);
+                    // only replace if word acutally changed
+                    if (!t.termText().equalsIgnoreCase(suggestions[i])) {
+                        sb.replace(t.startOffset(), t.endOffset(), suggestions[i]);
+                    }
+                }
+                return sb.toString();
+            } else {
+                return null;
+            }
+        }
+
+        void close() {
+            try {
+                spellIndexDirectory.close();
+            } catch (IOException e) {
+                // ignore
+            }
+            // urgh, the lucene spell checker cannot be closed explicitly.
+            // finalize will close the reader...
+            spellChecker = null;
+        }
+
+        /**
+         * Tokenizes the statement into words and tokens.
+         *
+         * @param statement the fulltext query statement.
+         * @param words     this list will be filled with the original words
+         *                  extracted from the statement.
+         * @param tokens    this list will be filled with the tokens parsed from
+         *                  the statement.
+         * @throws IOException if an error occurs while parsing the statement.
+         */
+        private void tokenize(String statement, List words, List tokens)
+                throws IOException {
+            TokenStream ts = handler.getTextAnalyzer().tokenStream(
+                    FieldNames.FULLTEXT, new StringReader(statement));
+            try {
+                Token t;
+                while ((t = ts.next()) != null) {
+                    String origWord = statement.substring(t.startOffset(), t.endOffset());
+                    if (t.getPositionIncrement() > 0) {
+                        words.add(t.termText());
+                        tokens.add(t);
+                    } else {
+                        // very simple implementation: use termText with length
+                        // closer to original word
+                        Token current = (Token) tokens.get(tokens.size() - 1);
+                        if (Math.abs(origWord.length() - current.termText().length()) >
+                                Math.abs(origWord.length() - t.termText().length())) {
+                            // replace current token and word
+                            words.set(words.size() - 1, t.termText());
+                            tokens.set(tokens.size() - 1, t);
+                        }
+                    }
+                }
+            } finally {
+                ts.close();
+            }
+        }
+
+        /**
+         * Checks the spelling of the passed <code>words</code> and returns a
+         * suggestion.
+         *
+         * @param words the words to check.
+         * @return a suggestion of correctly spelled <code>words</code> or
+         *         <code>null</code> if this spell checker thinks
+         *         <code>words</code> are spelled correctly.
+         * @throws IOException if an error occurs while spell checking.
+         */
+        private String[] check(String words[]) throws IOException {
+            refreshSpellChecker();
+            boolean hasSuggestion = false;
+            IndexReader reader = handler.getIndexReader();
+            try {
+                for (int retries = 0; retries < 100; retries++) {
+                    try {
+                        String[] suggestion = new String[words.length];
+                        for (int i = 0; i < words.length; i++) {
+                            String[] similar = spellChecker.suggestSimilar(words[i], 5, reader,
+                                    FieldNames.FULLTEXT, true);
+                            if (similar.length > 0) {
+                                suggestion[i] = similar[0];
+                                hasSuggestion = true;
+                            } else {
+                                suggestion[i] = words[i];
+                            }
+                        }
+                        if (hasSuggestion) {
+                            log.info("Successful after {} retries", new Integer(retries));
+                            return suggestion;
+                        } else {
+                            return null;
+                        }
+                    } catch (AlreadyClosedException e) {
+                        // it may happen that the index reader inside the
+                        // spell checker is closed while searching for
+                        // suggestions. this is actually a design flaw in the
+                        // lucene spell checker, but for now we simply retry
+                    }
+                }
+                // unsuccessful after retries
+                return null;
+            } finally {
+                reader.close();
+            }
+        }
+
+        /**
+         * Refreshes the underlying spell checker in a background thread.
+         * Synchronization is done on this <code>LuceneSpellChecker</code> instance.
+         * While the refresh takes place {@link #refreshing} is set to
+         * <code>true</code>.
+         */
+        private void refreshSpellChecker() {
+            if (lastRefresh + refreshInterval < System.currentTimeMillis()) {
+                synchronized (this) {
+                    if (refreshing) {
+                        return;
+                    } else {
+                        refreshing = true;
+                        Runnable refresh = new Runnable() {
+                            public void run() {
+                                try {
+                                    IndexReader reader = handler.getIndexReader();
+                                    try {
+                                        long time = System.currentTimeMillis();
+                                        Dictionary dict = new LuceneDictionary(
+                                                reader, FieldNames.FULLTEXT);
+                                        log.debug("Starting spell checker index refresh");
+                                        spellChecker.indexDictionary(dict);
+                                        time = System.currentTimeMillis() - time;
+                                        time = time / 1000;
+                                        log.info("Spell checker index refreshed in: {} s.",
+                                                new Long(time));
+                                    } finally {
+                                        reader.close();
+                                        synchronized (InternalSpellChecker.this) {
+                                            refreshing = false;
+                                        }
+                                    }
+                                } catch (IOException e) {
+                                    // ignore
+                                }
+                            }
+                        };
+                        new Thread(refresh, "SpellChecker Refresh").start();
+                        lastRefresh = System.currentTimeMillis();
+                    }
+                }
+            }
+        }
+    }
+}

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/src/main/java/org/apache/jackrabbit/core/query/lucene/spell/LuceneSpellChecker.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java Wed Oct 24 05:17:51 2007
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.query.lucene.spell;
+
+import junit.framework.TestCase;
+
+import javax.jcr.RepositoryException;
+import javax.jcr.Repository;
+import javax.jcr.Session;
+import javax.jcr.SimpleCredentials;
+import javax.jcr.Value;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.Query;
+import javax.jcr.query.RowIterator;
+import javax.jcr.query.Row;
+
+import org.apache.jackrabbit.core.TransientRepository;
+import org.apache.jackrabbit.core.config.RepositoryConfig;
+import org.apache.log4j.PropertyConfigurator;
+
+import java.io.InputStream;
+
+/**
+ * <code>SpellCheckerTest</code> performs some spell checker tests.
+ */
+public class SpellCheckerTest extends TestCase {
+
+    private static Repository repository;
+    private Session session;
+
+    protected void setUp() throws Exception {
+        super.setUp();
+        if (repository == null) {
+            ClassLoader cl = getClass().getClassLoader();
+            PropertyConfigurator.configure(cl.getResource("log4j.properties"));
+            InputStream in = cl.getResourceAsStream("repository.xml");
+            RepositoryConfig config;
+            try {
+                config = RepositoryConfig.create(in, "target/repo-home");
+            } finally {
+                in.close();
+            }
+            repository = new TransientRepository(config);
+            session = repository.login(new SimpleCredentials("user", "password".toCharArray()));
+            String text = "the quick brown fox jumps over the lazy dog";
+            session.getRootNode().setProperty("prop", text);
+            session.save();
+            // wait a couple of seconds, refresh interval in test config is 5 seconds
+            try {
+                Thread.sleep(5 * 1000);
+                // perform a dummy check
+                performCheck("quick", "quick");
+                // wait again because refresh is done asynchronous
+                Thread.sleep(1 * 1000);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        if (session == null) {
+            session = repository.login(new SimpleCredentials("user", "password".toCharArray()));
+        }
+    }
+
+    protected void tearDown() throws Exception {
+        if (session != null) {
+            session.logout();
+        }
+        session = null;
+        super.tearDown();
+    }
+
+    public void testSpellChecker() throws RepositoryException {
+        performCheck("quik", "quick");
+    }
+
+    public void testSpellCheckerComplexQuery() throws RepositoryException {
+        performCheck("quik OR (-foo bar)", "quick OR (-fox bar)");
+    }
+
+    public void testSpellCheckerCorrectWords() throws RepositoryException {
+        performCheck("quick", "quick");
+    }
+
+    protected void performCheck(String statement, String expected)
+            throws RepositoryException {
+        QueryManager qm = session.getWorkspace().getQueryManager();
+        Query query = qm.createQuery("/jcr:root[rep:spellcheck('" + statement +
+                "')]/(rep:spellcheck())", Query.XPATH);
+        RowIterator rows = query.execute().getRows();
+        assertEquals("no results returned", 1, rows.getSize());
+        Row r = rows.nextRow();
+        Value v = r.getValue("rep:spellcheck()");
+        if (statement.equals(expected)) {
+            assertNull("must not return a suggestion", v);
+        } else {
+            assertNotNull("no suggestion returned", v);
+            assertEquals("wrong suggestion returned", expected, v.getString());
+        }
+
+        query = qm.createQuery("select rep:spellcheck() from nt:base where " +
+                "jcr:path = '/' and spellcheck('" + statement + "')", Query.SQL);
+        rows = query.execute().getRows();
+        assertEquals("no results returned", 1, rows.getSize());
+        r = rows.nextRow();
+        v = r.getValue("rep:spellcheck()");
+        if (statement.equals(expected)) {
+            assertNull("must not return a suggestion", v);
+        } else {
+            assertNotNull("no suggestion returned", v);
+            assertEquals("wrong suggestion returned", expected, v.getString());
+        }
+    }
+}

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/java/org/apache/jackrabbit/core/query/lucene/spell/SpellCheckerTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties Wed Oct 24 05:17:51 2007
@@ -0,0 +1,36 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=INFO, file
+#log4j.rootLogger=DEBUG, stdout, file
+#log4j.rootLogger=ERROR, stdout, file
+
+log4j.logger.org.apache.jackrabbit.test=DEBUG
+
+# 'stdout' is set to be a ConsoleAppender.
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+
+# 'stdout' uses PatternLayout
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{dd.MM.yyyy HH:mm:ss} *%-5p* [%t] %c{1}: %m (%F, line %L)\n
+
+# 'file' is set to be a FileAppender.
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.File=jcr.log
+
+# 'file' uses PatternLayout.
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{dd.MM.yyyy HH:mm:ss} *%-5p* [%t] %c{1}: %m (%F, line %L)\n

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/log4j.properties
------------------------------------------------------------------------------
    svn:eol-style = native

Added: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml?rev=587868&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml (added)
+++ jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml Wed Oct 24 05:17:51 2007
@@ -0,0 +1,118 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  -->
+<!DOCTYPE Repository PUBLIC "-//The Apache Software Foundation//DTD Jackrabbit 1.2//EN"
+                            "http://jackrabbit.apache.org/dtd/repository-1.2.dtd">
+<!-- Example Repository Configuration File -->
+<Repository>
+    <!--
+        virtual file system where the repository stores global state
+        (e.g. registered namespaces, custom node types, etc.)
+    -->
+    <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
+        <param name="path" value="${rep.home}/repository"/>
+    </FileSystem>
+
+    <!--
+        security configuration
+    -->
+    <Security appName="Jackrabbit">
+        <!--
+            access manager:
+            class: FQN of class implementing the AccessManager interface
+        -->
+        <AccessManager class="org.apache.jackrabbit.core.security.SimpleAccessManager">
+            <!-- <param name="config" value="${rep.home}/access.xml"/> -->
+        </AccessManager>
+
+        <LoginModule class="org.apache.jackrabbit.core.security.SimpleLoginModule">
+           <!-- anonymous user name ('anonymous' is the default value) -->
+           <param name="anonymousId" value="anonymous"/>
+           <!--
+              default user name to be used instead of the anonymous user
+              when no login credentials are provided (unset by default)
+           -->
+           <!-- <param name="defaultUserId" value="superuser"/> -->
+        </LoginModule>
+    </Security>
+
+    <!--
+        location of workspaces root directory and name of default workspace
+    -->
+    <Workspaces rootPath="${rep.home}/workspaces" defaultWorkspace="default"/>
+    <!--
+        workspace configuration template:
+        used to create the initial workspace if there's no workspace yet
+    -->
+    <Workspace name="${wsp.name}">
+        <!--
+            virtual file system of the workspace:
+            class: FQN of class implementing the FileSystem interface
+        -->
+        <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
+            <param name="path" value="${wsp.home}"/>
+        </FileSystem>
+        <!--
+            persistence manager of the workspace:
+            class: FQN of class implementing the PersistenceManager interface
+        -->
+        <PersistenceManager class="org.apache.jackrabbit.core.persistence.db.DerbyPersistenceManager">
+          <param name="url" value="jdbc:derby:${wsp.home}/db;create=true"/>
+          <param name="schemaObjectPrefix" value="${wsp.name}_"/>
+        </PersistenceManager>
+        <!--
+            Search index and the file system it uses.
+            class: FQN of class implementing the QueryHandler interface
+        -->
+        <SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
+            <param name="path" value="${wsp.home}/index"/>
+            <param name="spellCheckerClass" value="org.apache.jackrabbit.core.query.lucene.spell.LuceneSpellChecker$FiveSecondsRefreshInterval"/>
+        </SearchIndex>
+    </Workspace>
+
+    <!--
+        Configures the versioning
+    -->
+    <Versioning rootPath="${rep.home}/version">
+        <!--
+            Configures the filesystem to use for versioning for the respective
+            persistence manager
+        -->
+        <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
+            <param name="path" value="${rep.home}/version" />
+        </FileSystem>
+
+        <!--
+            Configures the persistence manager to be used for persisting version state.
+            Please note that the current versioning implementation is based on
+            a 'normal' persistence manager, but this could change in future
+            implementations.
+        -->
+        <PersistenceManager class="org.apache.jackrabbit.core.persistence.db.DerbyPersistenceManager">
+          <param name="url" value="jdbc:derby:${rep.home}/version/db;create=true"/>
+          <param name="schemaObjectPrefix" value="version_"/>
+        </PersistenceManager>
+    </Versioning>
+
+    <!--
+        Search index for content that is shared repository wide
+        (/jcr:system tree, contains mainly versions)
+    -->
+    <SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
+        <param name="path" value="${rep.home}/repository/index"/>
+    </SearchIndex>
+</Repository>

Propchange: jackrabbit/sandbox/jackrabbit-spellchecker/src/test/resources/repository.xml
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message