roller-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject svn commit: r398712 [5/32] - in /incubator/roller/trunk/src/org/apache: ./ roller/ roller/business/ roller/business/hibernate/ roller/business/referrers/ roller/business/runnable/ roller/business/search/ roller/business/search/operations/ roller/busine...
Date Mon, 01 May 2006 22:23:34 GMT
Added: incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java Mon May  1 15:23:02 2006
@@ -0,0 +1,216 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * ReferrerQueueManagerImpl.java
+ *
+ * Created on December 16, 2005, 5:06 PM
+ */
+
+package org.apache.roller.business.referrers;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.RollerException;
+import org.apache.roller.business.runnable.ContinuousWorkerThread;
+import org.apache.roller.business.runnable.WorkerThread;
+import org.apache.roller.config.RollerConfig;
+import org.apache.roller.model.RollerFactory;
+
+
+/**
+ * The base implementation of the ReferrerQueueManager.
+ * 
+ * This class is implemented using the singleton pattern to ensure that only
+ * one instance exists at any given time.
+ *
+ * This implementation can be configured to handle referrers in 2 ways ...
+ *  1. synchronously.  referrers are processed immediately.
+ *  2. asynchronously.  referrers are queued for later processing.
+ *
+ * Users can control the referrer queue mode via properties in the static
+ * roller.properties configuration file.
+ *
+ * In asynchronous processing mode we start some number of worker threads which
+ * run continously to process any referrers that have been queued.  Each worker
+ * processes queued referrers until the queue is empty, then sleeps for a given
+ * amount of time.  The number of workers used and their sleep time can be set
+ * via properties of the static roller.properties file.
+ *
+ * @author Allen Gilliland
+ */
+public class ReferrerQueueManagerImpl implements ReferrerQueueManager {
+    
+    private static Log mLogger = LogFactory.getLog(ReferrerQueueManagerImpl.class);
+    
+    private static ReferrerQueueManager instance = null;
+    
+    private boolean asyncMode = false;
+    private int numWorkers = 1;
+    private int sleepTime = 10000;
+    private List workers = null;
+    private List referrerQueue = null;
+    
+    static {
+        instance = new ReferrerQueueManagerImpl();
+    }
+    
+    
+    // private because we are a singleton
+    private ReferrerQueueManagerImpl() {
+        mLogger.info("Initializing Referrer Queue Manager");
+        
+        // lookup config options
+        this.asyncMode = RollerConfig.getBooleanProperty("referrers.asyncProcessing.enabled");
+        
+        mLogger.info("Asynchronous referrer processing = "+this.asyncMode);
+        
+        if(this.asyncMode) {
+            
+            
+            String num = RollerConfig.getProperty("referrers.queue.numWorkers");
+            String sleep = RollerConfig.getProperty("referrers.queue.sleepTime");
+            
+            try {
+                this.numWorkers = Integer.parseInt(num);
+                
+                if(numWorkers < 1)
+                    this.numWorkers = 1;
+                
+            } catch(NumberFormatException nfe) {
+                mLogger.warn("Invalid num workers ["+num+"], using default");
+            }
+            
+            try {
+                // multiply by 1000 because we expect input in seconds
+                this.sleepTime = Integer.parseInt(sleep) * 1000;
+            } catch(NumberFormatException nfe) {
+                mLogger.warn("Invalid sleep time ["+sleep+"], using default");
+            }
+            
+            // create the processing queue
+            this.referrerQueue = Collections.synchronizedList(new ArrayList());
+            
+            // start up workers
+            this.workers = new ArrayList();
+            ContinuousWorkerThread worker = null;
+            QueuedReferrerProcessingJob job = null;
+            for(int i=0; i < this.numWorkers; i++) {
+                job = new QueuedReferrerProcessingJob();
+                worker = new ContinuousWorkerThread("ReferrerWorker"+i, job, this.sleepTime);
+                workers.add(worker);
+                worker.start();
+            }
+        }
+    }
+    
+    
+    /**
+     * Get access to the singleton instance.
+     */
+    public static ReferrerQueueManager getInstance() {
+        return instance;
+    }
+    
+    
+    /**
+     * Process an incoming referrer.
+     *
+     * If we are doing asynchronous referrer processing then the referrer will
+     * just go into the queue for later processing.  If not then we process it
+     * now.
+     */
+    public void processReferrer(IncomingReferrer referrer) {
+        
+        if(this.asyncMode) {
+            mLogger.debug("QUEUING: "+referrer.getRequestUrl());
+            
+            // add to queue
+            this.enqueue(referrer);
+        } else {
+            // process now
+            ReferrerProcessingJob job = new ReferrerProcessingJob();
+            
+            // setup input
+            HashMap inputs = new HashMap();
+            inputs.put("referrer", referrer);
+            job.input(inputs);
+            
+            // execute
+            job.execute();
+            
+            try {
+                // flush changes
+                RollerFactory.getRoller().flush();
+            } catch (RollerException ex) {
+                mLogger.error("ERROR commiting referrer", ex);
+            }
+        }
+        
+    }
+    
+    
+    /**
+     * Place a referrer in the queue.
+     */
+    public void enqueue(IncomingReferrer referrer) {
+        this.referrerQueue.add(referrer);
+        
+        if(this.referrerQueue.size() > 250) {
+            mLogger.warn("Referrer queue is rather full. queued="+this.referrerQueue.size());
+        }
+    }
+    
+    
+    /**
+     * Retrieve the next referrer in the queue.
+     */
+    public synchronized IncomingReferrer dequeue() {
+        
+        if(!this.referrerQueue.isEmpty()) {
+            return (IncomingReferrer) this.referrerQueue.remove(0);
+        }
+        
+        return null;
+    }
+    
+    
+    /**
+     * clean up.
+     */
+    public void shutdown() {
+        
+        if(this.workers != null && this.workers.size() > 0) {
+            mLogger.info("stopping all ReferrerQueue worker threads");
+            
+            // kill all of our threads
+            WorkerThread worker = null;
+            Iterator it = this.workers.iterator();
+            while(it.hasNext()) {
+                worker = (WorkerThread) it.next();
+                worker.interrupt();
+            }
+        }
+        
+    }
+    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png?rev=398712&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png?rev=398712&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java Mon May  1 15:23:02 2006
@@ -0,0 +1,96 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * ContinuousWorkerThread.java
+ *
+ * Created on December 20, 2005, 1:57 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
+/**
+ * A worker that performs a given job continuously.
+ *
+ * @author Allen Gilliland
+ */
+public class ContinuousWorkerThread extends WorkerThread {
+    
+    private static Log mLogger = LogFactory.getLog(ContinuousWorkerThread.class);
+    
+    // default sleep time is 10 seconds
+    long sleepTime = 10000;
+    
+    
+    public ContinuousWorkerThread(String id) {
+        super(id);
+    }
+    
+    
+    public ContinuousWorkerThread(String id, long sleep) {
+        super(id);
+        
+        this.sleepTime = sleep;
+    }
+    
+    
+    public ContinuousWorkerThread(String id, Job job) {
+        super(id, job);
+    }
+    
+    
+    public ContinuousWorkerThread(String id, Job job, long sleep) {
+        super(id, job);
+        
+        this.sleepTime = sleep;
+    }
+    
+    
+    /**
+     * Thread execution.
+     *
+     * We run forever.  Each time a job completes we sleep for 
+     * some amount of time before trying again.
+     *
+     * If we ever get interrupted then we quit.
+     */
+    public void run() {
+        
+        mLogger.info(this.id+" Started.");
+        
+        // run forever
+        while(true) {
+            
+            // execute our job
+            super.run();
+            
+            // job is done, lets sleep it off for a bit
+            try {
+                mLogger.debug(this.id+" SLEEPING for "+this.sleepTime+" milliseconds ...");
+                this.sleep(this.sleepTime);
+            } catch (InterruptedException e) {
+                mLogger.info(this.id+" INTERRUPT: "+e.getMessage());
+                break;
+            }
+        }
+    }
+    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java Mon May  1 15:23:02 2006
@@ -0,0 +1,52 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Job.java
+ *
+ * Created on December 16, 2005, 6:14 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import java.util.Map;
+
+/**
+ * A job to be executed.
+ *
+ * @author Allen Gilliland
+ */
+public interface Job {
+    
+    /**
+     * Execute the job.
+     */
+    public void execute();
+    
+    
+    /**
+     * Pass in input to be used for the job.
+     */
+    public void input(Map input);
+    
+    
+    /**
+     * Get any output from the job.
+     */
+    public Map output();
+    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java Mon May  1 15:23:02 2006
@@ -0,0 +1,99 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * WorkerThread.java
+ *
+ * Created on December 16, 2005, 6:12 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+
+
+/**
+ * A generic worker thread that knows how execute a Job.
+ *
+ * @author Allen Gilliland
+ */
+public class WorkerThread extends Thread {
+    
+    private static Log mLogger = LogFactory.getLog(WorkerThread.class);
+    
+    String id = null;
+    Job job = null;
+    
+    
+    /**
+     * A simple worker.
+     */
+    public WorkerThread(String id) {
+        this.id = id;
+    }
+    
+    
+    /**
+     * Start off with a job to do.
+     */
+    public WorkerThread(String id, Job job) {
+        this.id = id;
+        this.job = job;
+    }
+    
+    
+    /**
+     * Thread execution.
+     *
+     * We just execute the job we were given if it's non-null.
+     */
+    public void run() {
+        
+        // we only run once
+        if (this.job != null) {
+            // process job
+            try {
+                this.job.execute();
+            } catch(Throwable t) {
+                // oops
+                mLogger.error("Error executing job. "+
+                        "Worker = "+this.id+", "+
+                        "Job = "+this.job.getClass().getName(), t);
+            }
+            
+            // since this is a thread we have to make sure that we tidy up ourselves
+            Roller roller = RollerFactory.getRoller();
+            roller.release();
+        }
+        
+    }
+    
+    
+    /**
+     * Set the job for this worker.
+     */
+    public void setJob(Job newJob) {
+        mLogger.debug("NEW JOB: "+newJob.getClass().getName());
+        
+        // set the job
+        this.job = newJob;
+    }
+    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java Mon May  1 15:23:02 2006
@@ -0,0 +1,51 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 19, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search;
+
+
+/**
+ * @author aim4min
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public final class FieldConstants
+{
+    //~ Static fields/initializers =============================================
+
+    public static final String ANCHOR = "anchor";
+    public static final String UPDATED = "updated";
+    public static final String ID = "id";
+    public static final String USERNAME = "username";
+    public static final String CATEGORY = "cat";
+    public static final String TITLE = "title";
+    public static final String PUBLISHED = "published";
+    public static final String CONTENT = "content";
+    public static final String CONTENT_STORED = "content_stored";
+    public static final String C_CONTENT = "comment";
+    public static final String C_EMAIL = "email";
+    public static final String C_NAME = "name";
+    public static final String CONSTANT = "constant";
+    public static final String CONSTANT_V = "v"; //must be lowercase, or match the transform rules of the analyzer
+    public static final String WEBSITE_HANDLE = "handle";
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java Mon May  1 15:23:02 2006
@@ -0,0 +1,66 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 20, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+/**
+ * @author aim4min
+ *
+ * Class containing helper methods. 
+ */
+public class IndexUtil {
+
+	/**
+	 * Create a lucene term from the first token of the input string. 
+	 * 
+	 * @param field The lucene document field to create a term with
+	 * @param input The input you wish to convert into a term
+	 * @return Lucene search term
+	 */
+	public static final Term getTerm(String field, String input) {
+		if (input==null || field==null) return null;
+		Analyzer analyer = IndexManagerImpl.getAnalyzer();
+		TokenStream tokens = analyer.tokenStream(field,
+			new StringReader(input));
+		
+		Token token = null;
+		Term term = null;
+		try {
+			token = tokens.next();
+		} catch (IOException e) {}
+		if (token!=null) {
+			String termt = token.termText();
+			term = new Term(field,termt);
+		}
+		return term;
+	}
+
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,90 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.pojos.WeblogEntryData;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that adds a new log entry into the index.
+ */
+public class AddEntryOperation extends WriteToIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(AddEntryOperation.class);
+
+    //~ Instance fields ========================================================
+
+    private WeblogEntryData data;
+
+    //~ Constructors ===========================================================
+
+    /**
+     * Adds a web log entry into the index.
+     */
+    public AddEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+    {
+        super(mgr);
+        this.data = data;
+    }
+
+    //~ Methods ================================================================
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {    	
+        IndexWriter writer = beginWriting();
+        Roller roller = RollerFactory.getRoller();
+        try
+        {
+            if (writer != null)
+            {
+                writer.addDocument(getDocument(data));
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.error("Problems adding doc to index", e);
+        }
+        finally
+        {
+            if (roller != null) roller.release();
+            endWriting();
+        }    	
+    }
+    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,190 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.pojos.CommentData;
+import org.apache.roller.pojos.WeblogCategoryData;
+import org.apache.roller.pojos.WeblogEntryData;
+import org.apache.roller.util.Utilities;
+import org.apache.roller.config.RollerConfig;
+
+/**
+ * @author aim4min
+ *         <p/>
+ *         This is the base class for all index operation. These operations include:
+ *         <p/>
+ *         SearchOperation AddWeblogOperation RemoveWeblogOperation
+ *         RebuildUserIndexOperation
+ */
+public abstract class IndexOperation implements Runnable {
+    private static Log mLogger = LogFactory.getFactory().getInstance(IndexOperation.class);
+
+    //~ Instance fields
+    // ========================================================
+    protected IndexManagerImpl manager;
+
+    private IndexReader reader;
+
+    private IndexWriter writer;
+
+    //~ Constructors
+    // ===========================================================
+
+    public IndexOperation(IndexManagerImpl manager) {
+        this.manager = manager;
+    }
+
+    //~ Methods
+    // ================================================================
+
+    protected Document getDocument(WeblogEntryData data) {
+
+        // Actual comment content is indexed only if search.index.comments
+        // is true or absent from the (static) configuration properties.
+        // If false in the configuration, comments are treated as if empty.
+        boolean indexComments = RollerConfig.getBooleanProperty("search.index.comments", true);
+
+        String commentContent = "";
+        String commentEmail = "";
+        String commentName = "";
+        if (indexComments) {
+            List comments = data.getComments();
+            if (comments != null) {
+                StringBuffer commentEmailBuf = new StringBuffer();
+                StringBuffer commentContentBuf = new StringBuffer();
+                StringBuffer commentNameBuf = new StringBuffer();
+                for (Iterator cItr = comments.iterator(); cItr.hasNext();) {
+                    CommentData comment = (CommentData) cItr.next();
+                    if (comment.getSpam() == null || !comment.getSpam().booleanValue()) {
+                        if (comment.getContent() != null) {
+                            commentContentBuf.append(comment.getContent());
+                            commentContentBuf.append(",");
+                        }
+                        if (comment.getEmail() != null) {
+                            commentEmailBuf.append(comment.getEmail());
+                            commentEmailBuf.append(",");
+                        }
+                        if (comment.getName() != null) {
+                            commentNameBuf.append(comment.getName());
+                            commentNameBuf.append(",");
+                        }
+                    }
+                }
+                commentEmail = commentEmailBuf.toString();
+                commentContent = commentContentBuf.toString();
+                commentName = commentNameBuf.toString();
+            }
+        }
+
+        Document doc = new Document();
+
+        doc.add(Field.Keyword(FieldConstants.ID, data.getId()));
+
+        doc.add(Field.Keyword(FieldConstants.WEBSITE_HANDLE, data.getWebsite().getHandle()));
+
+        doc.add(Field.UnIndexed(FieldConstants.ANCHOR, data.getAnchor()));
+        doc.add(Field.Text(FieldConstants.USERNAME, data.getCreator().getUserName()));
+        doc.add(Field.Text(FieldConstants.TITLE, data.getTitle()));
+
+        // index the entry text, but don't store it - moved to end of block
+        doc.add(Field.UnStored(FieldConstants.CONTENT, data.getText()));
+
+        // store an abbreviated version of the entry text, but don't index
+        doc.add(Field.UnIndexed(FieldConstants.CONTENT_STORED, Utilities
+                .truncateNicely(Utilities.removeHTML(data.getText()), 240, 260, "...")));
+
+        doc.add(Field.Keyword(FieldConstants.UPDATED, data.getUpdateTime()
+                .toString()));
+        doc.add(Field.Keyword(FieldConstants.PUBLISHED, data.getPubTime()
+                .toString()));
+
+        // index Comments
+        doc.add(Field.UnStored(FieldConstants.C_CONTENT, commentContent));
+        doc.add(Field.UnStored(FieldConstants.C_EMAIL, commentEmail));
+        doc.add(Field.UnStored(FieldConstants.C_NAME, commentName));
+
+        doc.add(Field.UnStored(FieldConstants.CONSTANT, FieldConstants.CONSTANT_V));
+
+        // index Category
+        WeblogCategoryData categorydata = data.getCategory();
+        Field category = (categorydata == null) ? Field.UnStored(FieldConstants.CATEGORY, "") : Field.Text(FieldConstants.CATEGORY, categorydata.getName());
+        doc.add(category);
+
+        return doc;
+    }
+
+    protected IndexReader beginDeleting() {
+        try {
+            reader = IndexReader.open(manager.getIndexDirectory());
+        } catch (IOException e) {
+        }
+
+        return reader;
+    }
+
+    protected void endDeleting() {
+        if (reader != null) {
+            try {
+                reader.close();
+            } catch (IOException e) {
+            }
+        }
+    }
+
+    protected IndexWriter beginWriting() {
+        try {
+            writer = new IndexWriter(manager.getIndexDirectory(), IndexManagerImpl.getAnalyzer(), false);
+        } catch (IOException e) {
+            mLogger.error("ERROR creating writer");
+        }
+
+        return writer;
+    }
+
+    protected void endWriting() {
+        if (writer != null) {
+            try {
+                writer.close();
+            } catch (IOException e) {
+                mLogger.error("ERROR closing writer", e);
+            }
+        }
+    }
+
+    public void run() {
+        doRun();
+    }
+
+    protected abstract void doRun();
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,109 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.pojos.WeblogEntryData;
+
+/**
+ * @author aim4min
+ *
+ * An operation that adds a new log entry into the index.
+ */
+public class ReIndexEntryOperation extends WriteToIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(AddEntryOperation.class);
+
+    //~ Instance fields ========================================================
+
+    private WeblogEntryData data;
+
+    //~ Constructors ===========================================================
+
+    /**
+     * Adds a web log entry into the index.
+     */
+    public ReIndexEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+    {
+        super(mgr);
+        this.data = data;
+    }
+
+    //~ Methods ================================================================
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {    	
+        IndexReader reader = beginDeleting();
+        try
+        {
+            if (reader != null)
+            {
+                Term term = new Term(FieldConstants.ID, data.getId());
+                reader.delete(term);
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.error("Error deleting doc from index", e);
+        }
+        finally
+        {
+            endDeleting();
+        }
+        
+        IndexWriter writer = beginWriting();
+        Roller roller = RollerFactory.getRoller();
+        try
+        {
+            if (writer != null)
+            {
+                writer.addDocument(getDocument(data));
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.error("Problems adding doc to index", e);
+        }
+        finally
+        {
+            if (roller != null) roller.release();
+            endWriting();
+        }    	
+    }    
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,64 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+
+package org.apache.roller.business.search.operations;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.business.IndexManagerImpl;
+
+/**
+ * @author aim4min
+ */
+public abstract class ReadFromIndexOperation extends IndexOperation
+{
+
+    /**
+     * @param manager
+     */
+    public ReadFromIndexOperation(IndexManagerImpl mgr)
+    {
+        super(mgr);
+    }
+
+    private static Log mLogger = LogFactory.getFactory().getInstance(
+            ReadFromIndexOperation.class);
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see java.lang.Runnable#run()
+     */
+    public final void run()
+    {
+        try
+        {
+            manager.getReadWriteLock().readLock().acquire();
+            doRun();
+        }
+        catch (InterruptedException e)
+        {
+            mLogger.info("Error acquiring read lock on index", e);
+        }
+        finally
+        {
+            manager.getReadWriteLock().readLock().release();
+        }
+    }
+
+}
\ No newline at end of file

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,173 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.roller.RollerException;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.model.WeblogManager;
+import org.apache.roller.pojos.WeblogEntryData;
+import org.apache.roller.pojos.WebsiteData;
+
+
+/**
+ * @author aim4min
+ *
+ * An index operation that rebuilds a given users index (or all indexes.)
+ */
+public class RebuildWebsiteIndexOperation extends WriteToIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(RebuildWebsiteIndexOperation.class);
+
+    //~ Instance fields ========================================================
+
+    private WebsiteData website;
+
+    //~ Constructors ===========================================================
+
+    /**
+     * Create a new operation that will recreate an index.
+     *
+     * @param website The website to rebuild the index for, or null for all users.
+     */
+    public RebuildWebsiteIndexOperation(IndexManagerImpl mgr, WebsiteData website)
+    {
+        super(mgr);
+        this.website = website;
+    }
+
+    //~ Methods ================================================================
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {
+        Date start = new Date();
+
+        IndexReader reader = beginDeleting();
+
+        try
+        {
+            if (reader != null)
+            {
+                Term tWebsite = null;
+                if (website != null) 
+                {
+                    tWebsite = IndexUtil.getTerm(
+                        FieldConstants.WEBSITE_HANDLE, website.getHandle());
+                }
+                if (tWebsite != null)
+                {
+                    reader.delete(tWebsite);
+                }
+                else
+                {
+                    Term all =
+                        IndexUtil.getTerm(FieldConstants.CONSTANT,
+                                          FieldConstants.CONSTANT_V);
+                    reader.delete(all);
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.info("Problems deleting doc from index", e);
+        }
+        finally
+        {
+            endDeleting();
+        }
+
+        IndexWriter writer = beginWriting();
+
+        Roller roller = RollerFactory.getRoller();
+        try
+        {
+            if (writer != null)
+            {
+                WeblogManager weblogManager = roller.getWeblogManager();
+
+                List entries = weblogManager .getWeblogEntries(
+                    website,                   // userName
+                    null,                      // startDate
+                    new Date(),                // endDate (don't index 'future' entries)
+                    null,                      // catName
+                    WeblogEntryData.PUBLISHED, // status
+                    null,                      // sortby (null mean pubTime)
+                    null);                     // maxEntries (null to get 'em all)
+
+                for (Iterator wbItr = entries.iterator(); wbItr.hasNext();) 
+                {
+                    WeblogEntryData entry = (WeblogEntryData) wbItr.next();
+                    writer.addDocument(getDocument(entry));
+                    mLogger.debug(
+                       MessageFormat.format("Indexed entry {0}: {1}",
+                       new Object[] {entry.getPubTime(), entry.getAnchor()}));
+                }
+                // release the database connection
+                roller.release();
+            }
+        }
+        catch (Exception e)
+        {
+            mLogger.error("ERROR adding doc to index", e);
+        }
+        finally
+        {
+            endWriting();
+            if (roller != null) roller.release();
+        }
+
+        Date end = new Date();
+        double length = (end.getTime() - start.getTime()) / (double) 1000;
+
+        if (website == null)
+        {
+            mLogger.info(
+               "Completed rebuilding index for all users in '" + length + "' secs");
+        }
+        else
+        {
+            mLogger.info("Completed rebuilding index for website handle: '" +
+                 website.getHandle() + "' in '" + length + "' seconds");
+        }
+    }
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,90 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ *
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.pojos.WeblogEntryData;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that removes the weblog from the index.
+ */
+public class RemoveEntryOperation extends WriteToIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(RemoveEntryOperation.class);
+
+    //~ Instance fields ========================================================
+
+    private WeblogEntryData data;
+
+    //~ Constructors ===========================================================
+
+    /**
+     *
+     */
+    public RemoveEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+    {
+        super(mgr);
+        this.data = data;
+    }
+
+    //~ Methods ================================================================
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {
+        IndexReader reader = beginDeleting();
+        try
+        {
+            if (reader != null)
+            {
+                Term term = new Term(FieldConstants.ID, data.getId());
+                reader.delete(term);
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.error("Error deleting doc from index", e);
+        }
+        finally
+        {
+            endDeleting();
+        }
+    }
+
+ 
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,112 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.pojos.WebsiteData;
+
+
+/**
+ * @author aim4min
+ *
+ * An index operation that rebuilds a given users index (or all indexes.)
+ */
+public class RemoveWebsiteIndexOperation extends WriteToIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(RemoveWebsiteIndexOperation.class);
+
+    //~ Instance fields ========================================================
+
+    private WebsiteData website;
+
+    //~ Constructors ===========================================================
+
+    /**
+     * Create a new operation that will recreate an index.
+     *
+     * @param website The website to rebuild the index for, or null for all sites.
+     */
+    public RemoveWebsiteIndexOperation(IndexManagerImpl mgr, WebsiteData website)
+    {
+        super(mgr);
+        this.website = website;
+    }
+
+    //~ Methods ================================================================
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {
+        Date start = new Date();
+        IndexReader reader = beginDeleting();
+        try
+        {
+            if (reader != null)
+            {
+                String handle = null;
+                if (website != null)
+                {
+                    handle = website.getHandle();
+                }
+                Term tHandle =
+                    IndexUtil.getTerm(FieldConstants.WEBSITE_HANDLE, handle);
+
+                if (tHandle != null)
+                {
+                    reader.delete(tHandle);
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            mLogger.info("Problems deleting doc from index", e);
+        }
+        finally
+        {
+            endDeleting();
+        }
+
+        Date end = new Date();
+        double length = (end.getTime() - start.getTime()) / (double) 1000;
+
+        if (website != null)
+        {
+            mLogger.info("Completed deleting indices for website '" +
+                    website.getName() + "' in '" + length + "' seconds");
+        }
+    }
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,179 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 18, 2003
+ * Authored by: Mindaugas Idzelis  (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryParser.MultiFieldQueryParser;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.model.IndexManager;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that searches the index.
+ */
+public class SearchOperation extends ReadFromIndexOperation
+{
+    //~ Static fields/initializers =============================================
+
+    private static Log mLogger =
+        LogFactory.getFactory().getInstance(SearchOperation.class);
+        
+    private static String[] SEARCH_FIELDS = new String[]{
+        FieldConstants.CONTENT, FieldConstants.TITLE, 
+        FieldConstants.C_CONTENT, FieldConstants.CATEGORY
+    };
+    
+    private static Sort SORTER = new Sort( new SortField(
+        FieldConstants.PUBLISHED, SortField.STRING, true) );
+
+    //~ Instance fields ========================================================
+
+    private String term;
+    private String websiteHandle;
+    private String category;
+    private Hits searchresults;
+    private String parseError;
+
+    //~ Constructors ===========================================================
+
+    /**
+     * Create a new operation that searches the index.
+     */
+    public SearchOperation(IndexManager mgr)
+    {
+        // TODO: finish moving  IndexManager to backend, so this cast is not needed
+        super((IndexManagerImpl)mgr); 
+    }
+
+    //~ Methods ================================================================
+
+    public void setTerm(String term)
+    {
+        this.term = term;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Runnable#run()
+     */
+    public void doRun()
+    {
+        searchresults = null;
+      		
+        IndexSearcher searcher = null;
+
+        try
+        {
+            IndexReader reader = manager.getSharedIndexReader();
+            searcher = new IndexSearcher(reader);
+
+            Query query =
+                MultiFieldQueryParser.parse(
+                    term, SEARCH_FIELDS, new StandardAnalyzer());
+
+            Term tUsername =
+                IndexUtil.getTerm(FieldConstants.WEBSITE_HANDLE, websiteHandle);
+
+            if (tUsername != null)
+            {
+                BooleanQuery bQuery = new BooleanQuery();
+                bQuery.add(query, true, false);
+                bQuery.add(new TermQuery(tUsername), true, false);
+                query = bQuery;
+            }
+            
+            Term tCategory =
+                IndexUtil.getTerm(FieldConstants.CATEGORY, category);
+
+            if (tCategory != null)
+            {
+                BooleanQuery bQuery = new BooleanQuery();
+                bQuery.add(query, true, false);
+                bQuery.add(new TermQuery(tCategory), true, false);
+                query = bQuery;
+            }
+            searchresults = searcher.search(query, null/*Filter*/, SORTER);
+        }
+        catch (IOException e)
+        {
+            mLogger.error("Error searching index", e);
+            parseError = e.getMessage();
+        }
+        catch (ParseException e)
+        {
+            // who cares?
+            parseError = e.getMessage();
+        }
+        // don't need to close the reader, since we didn't do any writing!
+    }
+
+    public Hits getResults()
+    {
+        return searchresults;
+    }
+    
+    public int getResultsCount()
+    {
+        if (searchresults == null) return -1;
+        
+        return searchresults.length();
+    }
+    
+    public String getParseError()
+    {
+        return parseError;
+    }
+
+    /**
+     * @param string
+     */
+    public void setWebsiteHandle(String websiteHandle)
+    {
+        this.websiteHandle = websiteHandle;
+    }
+
+    /**
+     * @param parameter
+     */
+    public void setCategory(String category)
+    {
+        this.category = category;
+    }
+
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java Mon May  1 15:23:02 2006
@@ -0,0 +1,71 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Aug 12, 2003
+ *
+ * To change the template for this generated file go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+package org.apache.roller.business.search.operations;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.business.IndexManagerImpl;
+
+/**
+ * @author aim4min
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public abstract class WriteToIndexOperation extends IndexOperation {
+
+	/**
+     * @param manager
+     */
+    public WriteToIndexOperation(IndexManagerImpl mgr)
+    {
+        super(mgr);
+    }
+
+    private static Log mLogger =
+		LogFactory.getFactory().getInstance(WriteToIndexOperation.class);
+		  
+	/* (non-Javadoc)
+	 * @see java.lang.Runnable#run()
+	 */
+	public void run() 
+    {
+		try 
+        {
+            manager.getReadWriteLock().writeLock().acquire();
+            mLogger.info("Starting search index operation");
+            doRun();
+            mLogger.info("Search index operation complete");
+		} 
+        catch (InterruptedException e) 
+        {
+			mLogger.error("Error acquiring write lock on index", e);
+		} 
+        finally 
+        {
+			manager.getReadWriteLock().writeLock().release();
+		}
+        manager.resetSharedReader();
+	}
+}

Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html Mon May  1 15:23:02 2006
@@ -0,0 +1,26 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  The ASF licenses this file to You
+  under the Apache License, Version 2.0 (the "License"); you may not
+  use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.  For additional information regarding
+  copyright in this work, please see the NOTICE file in the top level
+  directory of this distribution.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+  <title></title>
+</head>
+<body>
+Lucene-based search operations to be executed by ThreadManager.<br>
+</body>
+</html>

Added: incubator/roller/trunk/src/org/apache/roller/business/search/package.html
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/package.html?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/package.html (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/package.html Mon May  1 15:23:02 2006
@@ -0,0 +1,26 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  The ASF licenses this file to You
+  under the Apache License, Version 2.0 (the "License"); you may not
+  use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.  For additional information regarding
+  copyright in this work, please see the NOTICE file in the top level
+  directory of this distribution.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+  <title></title>
+</head>
+<body>
+Utilities needed by implementation of search interface.
+</body>
+</html>

Added: incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java Mon May  1 15:23:02 2006
@@ -0,0 +1,332 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+*  contributor license agreements.  The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.  For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+
+package org.apache.roller.business.utils;
+
+import java.io.FileInputStream;
+import java.sql.*;
+import java.util.*;
+
+/**
+ * Roller database consistency checker.<br />
+ * Don't run this unless you know what you are doing!</br >
+ * 
+ * <p>Configuration:<br />
+ * 
+ * Program looks in current directory for db.properties file with database
+ * connection properties driverClassName and connectionUrl. 
+ * 
+ * Program expects JDBC driver jar to be on classpath.</p>
+ * 
+ * <p>Usage:<br />
+ * 
+ * java -cp ./WEB-INF/lib/rollerbeans.jar org.apache.roller.business.utils.ConsistencyCheck<br />
+ * 
+ * <br />Options:<br />
+ * -v Verbose<br />
+ * -purge Delete orphans</p>
+ */
+public class ConsistencyCheck 
+{
+    /** 
+     * Consistency checker, find and optionally delete orphans. 
+     */
+    public static void main(String[] args) throws Exception
+    {
+        Properties props = new Properties();
+        props.load(new FileInputStream("rollerdb.properties"));
+        Connection con = createConnection(props,"");
+        
+        boolean delete = false;
+        boolean verbose = false;
+        if (args.length > 0) 
+        {
+            if ("-purge".equals(args[0])) 
+            {
+                delete = true;
+            }
+            else if ("-v".equals(args[0]))
+            {
+                verbose = true;
+            }
+        }        
+        
+        findAndDeleteOrphans(con, delete, verbose);
+    }
+    
+    /** 
+     * Create connection based on properties:<br/>
+     * - driverClassName<br/>
+     * - connectionUrl<br/>
+     * - userName<br/>
+     * - password<br/>
+     */
+    public static Connection createConnection(Properties props, String prefix) 
+        throws Exception
+    {
+        Connection con = null;
+        if (prefix == null) 
+        {
+            prefix = "";
+        }
+        String driverClassName = props.getProperty(prefix+"driverClassName");
+        String connectionUrl = props.getProperty(prefix+"connectionUrl");
+        String userName = props.getProperty(prefix+"userName");
+        String password = props.getProperty(prefix+"password");
+        
+        Class.forName(driverClassName);
+        if (userName != null && password != null)
+        {
+           con = DriverManager.getConnection(connectionUrl, userName, password);
+        }
+        else
+        {
+           con = DriverManager.getConnection(connectionUrl);
+        }
+        return con;
+    }
+    
+    /** Find and optionally delete all safely deletable orphans. */
+    public static void findAndDeleteOrphans(Connection con, boolean delete, boolean verbose) 
+        throws SQLException
+    {
+        // websites with bad user?
+        findOrphans(con, "website", "userid", "rolleruser", delete, verbose);
+        
+        // userroles with bad user?
+        findOrphans(con, "userrole", "userid", "rolleruser", delete, verbose);
+        
+        // folders with bad website?
+        findOrphans(con, "folder", "websiteid", "website", delete, verbose);
+        
+        // bookmarks with bad folder?
+        findOrphans(con, "bookmark", "folderid", "folder", delete, verbose);
+        
+        // weblogcategories with bad website?
+        findOrphans(con, "weblogcategory", "websiteid", "website", delete, verbose);
+        
+        // weblogcategoryassocs with bad category?
+        findOrphans(con, "weblogcategoryassoc", "categoryid", "weblogcategory", delete, verbose);
+        
+        // weblog entries with bad website?
+        findOrphans(con, "weblogentry", "websiteid", "website", delete, verbose);
+                
+        // comments with bad weblogentry?
+        findOrphans(con, "comment", "entryid", "weblogentry", delete, verbose);
+        
+        // Referers with bad website?
+        findOrphans(con, "referer", "websiteid", "website", delete, verbose);
+        
+        // Referers with bad website?
+        findOrphans(con, "referer", "entryid", "weblogentry", delete, verbose);              
+        
+        if (delete)
+        {
+            correctWeblogEntries(con);
+            correctWebsites(con);
+            correctFolderTrees(con, delete);
+        }
+    }
+    
+    /**
+     * @param con
+     * @param delete
+     */
+    private static void correctFolderTrees(Connection con, boolean delete) throws SQLException
+    {
+        PreparedStatement rootStatement = con.prepareStatement(
+            "select a.id from folder as f, folderassoc as a where "+
+            "f.websiteid=? and f.id=a.folderid and "+
+            "a.relation='PARENT' and a.ancestorid is null");
+        PreparedStatement childrenStatement = con.prepareStatement(
+            "select id from folderassoc where ancestorid=?");
+
+        // loop through all websites
+        Statement websitesStatement = con.createStatement();
+        ResultSet websitesResultSet = 
+            websitesStatement.executeQuery("select id from website");
+        while (websitesResultSet.next()) 
+        {
+            String websiteId = websitesResultSet.getString(1);
+            //debug("Website "+websiteId);
+            
+            // find root folder(s)
+            List rootIds = new LinkedList();
+            rootStatement.clearParameters();
+            rootStatement.setString(1, websiteId);
+            ResultSet rootResultSet = rootStatement.executeQuery();
+            while (rootResultSet.next())
+            {
+                rootIds.add(rootResultSet.getString(1));
+            }
+            if (rootIds.size() > 1) 
+            {
+                // too many roots, need to figure out which are bogus
+                Iterator rootIter = rootIds.iterator();
+                while (rootIter.hasNext())
+                {
+                    String rootId = (String)rootIter.next();
+                    childrenStatement.clearParameters();
+                    childrenStatement.setString(1, rootId);
+                    ResultSet childrenResultSet = childrenStatement.executeQuery();
+                    List childIds = new LinkedList();
+                    while (childrenResultSet.next()) 
+                    {
+                        childIds.add(childrenResultSet.getString(1));
+                    }
+                    if (childIds.size() == 0)
+                    {
+                        debug("Folder "+rootId+" in website "+websiteId+"is a bogus root folder!");
+                    }
+                }
+            }
+            else if (rootIds.size() == 0)
+            {
+                debug("Website "+websiteId+" has no root folder!");
+            }
+        }
+    }
+
+    private static void correctWeblogEntries(Connection con) throws SQLException
+    {
+        List entries = findOrphans(con, "weblogentry", "categoryid", "weblogcategory", false, false);
+        Iterator entryIter = entries.iterator();
+        while (entryIter.hasNext())
+        {
+            String entryid = (String) entryIter.next();
+            Statement websiteSt = con.createStatement();
+            ResultSet websiteRs = websiteSt.executeQuery(
+                "select websiteid from weblogentry where id="+entryid);
+            websiteRs.first();
+            String websiteid = websiteRs.getString(0);
+            
+            String rootid = getRootCategoryId(con, websiteid);
+            Statement st = con.createStatement();
+            st.executeUpdate("update weblogentry set categoryid='"+rootid+"' "
+                           +" where id='"+entryid+"'");
+        }   
+    }
+    
+    public static void correctWebsites(Connection con) throws SQLException
+    {
+        List websites = findOrphans(con, "website", "defaultcatid", "weblogcategory", false, false);
+        Iterator websiteIter = websites.iterator();
+        while (websiteIter.hasNext())
+        {
+            String websiteid = (String) websiteIter.next();
+            String rootid = getRootCategoryId(con, websiteid);
+            Statement st = con.createStatement();
+            st.executeUpdate("update website set defaultcatid='"+rootid+"' "
+                    +" where id='"+websiteid+"'");
+        }
+    
+        websites = findOrphans(con, "website", "bloggercatid", "weblogcategory", false, false);
+        websiteIter = websites.iterator();
+        while (websiteIter.hasNext())
+        {
+            String websiteid = (String) websiteIter.next();
+            String rootid = getRootCategoryId(con, websiteid);
+            Statement st = con.createStatement();
+            st.executeUpdate("update website set bloggercatid='"+rootid+"' "
+                            +"where id='"+websiteid+"'");
+        }
+    }
+    
+    public static String getRootCategoryId(Connection con, String websiteid)
+        throws SQLException
+    {
+        Statement st = con.createStatement();
+        String query = 
+            "select c.id from weblogcategory as c, weblogcategoryassoc as a "
+            +"where a.categoryid=c.id and a.ancestorid is null "
+            +"and c.websiteid ='"+websiteid+"'";
+        //System.out.println(query);
+        ResultSet rs = st.executeQuery(query);        
+        rs.next();
+        return rs.getString(1);
+    }
+    
+    /** Find orphans, records in a manytable that refer to a onetable that 
+     * no longer exists.
+     * @param con       Database connection to be used.
+     * @param manytable Name of the manytable.
+     * @param fkname    Name of the foreign key field in the manytable.
+     * @param onetable  Name of the onetable.
+     * @param delete    True if orphans in manytable are to be deleted.
+     * @return          List of orphans found (will be empty if delete is true.
+     * @throws SQLException
+     */
+    public static List findOrphans(
+        Connection con, String manytable, String fkname, String onetable, boolean delete, boolean verbose) 
+        throws SQLException
+    {
+        List orphans = new LinkedList();
+        
+        Statement stall = con.createStatement();
+        ResultSet rsall = stall.executeQuery(
+            "select id,"+fkname+" as fk from "+manytable);
+        while (rsall.next()) 
+        {
+            String id = rsall.getString("id");
+            String fk = rsall.getString("fk");
+            if (fk != null)
+            {
+                Statement stone = con.createStatement();
+                ResultSet rsone = stone.executeQuery(
+                    "select id from "+onetable+" where id='"+fk+"' limit 1");
+                if (!rsone.next()) 
+                {
+                    orphans.add(id);  
+                    System.out.println("   Found orphan in "+manytable+" id="+id); 
+                }                
+            }
+        } 
+        
+        if (!delete)
+        {
+            debug("Orphans found in "+manytable+" = "+orphans.size());
+            if (verbose)
+            {
+                Iterator iter = orphans.iterator();
+                while (iter.hasNext())
+                {
+                    String id = (String) iter.next();
+                    debug("   "+manytable+" id="+id);
+                }
+            }
+        }
+        else
+        {
+            debug("Deleting orphans found in "+manytable+" count = "+orphans.size());            
+            Iterator iter = orphans.iterator();
+            while (iter.hasNext())
+            {
+                String id = (String) iter.next();
+                Statement stdel = con.createStatement();
+                stdel.executeUpdate("delete from "+manytable+" where id='"+id+"'");
+            }
+            orphans = new LinkedList();
+        }
+        return orphans;
+    }
+    
+    private static void debug(String msg) 
+    {
+        System.out.println("DEBUG: "+msg);   
+    }
+}



Mime
View raw message