cocoon-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vgritse...@apache.org
Subject cvs commit: xml-cocoon2/src/scratchpad/src/org/apache/cocoon/transformation LuceneTransformer.java
Date Fri, 22 Nov 2002 16:56:34 GMT
vgritsenko    2002/11/22 08:56:34

  Added:       src/scratchpad/src/org/apache/cocoon/transformation
                        LuceneTransformer.java
  Log:
  Add LueceneTransformer. It allows to
  create or update several indeces via <lucene:index> tag.
  Each index will contain set of <lucene:document>s, and content
  of each document should be inside this document element.
  (this version of transformer does not perform crawling/retreival of documents).
  Transformer is very useful to generate index of database driven site.
  
  Revision  Changes    Path
  1.1                  xml-cocoon2/src/scratchpad/src/org/apache/cocoon/transformation/LuceneTransformer.java
  
  Index: LuceneTransformer.java
  ===================================================================
  /*
  
   ============================================================================
                     The Apache Software License, Version 1.1
   ============================================================================
  
   Copyright (C) 1999-2002 The Apache Software Foundation. All rights reserved.
  
   Redistribution and use in source and binary forms, with or without modifica-
   tion, are permitted provided that the following conditions are met:
  
   1. Redistributions of  source code must  retain the above copyright  notice,
      this list of conditions and the following disclaimer.
  
   2. Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.
  
   3. The end-user documentation included with the redistribution, if any, must
      include  the following  acknowledgment:  "This product includes  software
      developed  by the  Apache Software Foundation  (http://www.apache.org/)."
      Alternately, this  acknowledgment may  appear in the software itself,  if
      and wherever such third-party acknowledgments normally appear.
  
   4. The names "Apache Cocoon" and  "Apache Software Foundation" must  not  be
      used to  endorse or promote  products derived from  this software without
      prior written permission. For written permission, please contact
      apache@apache.org.
  
   5. Products  derived from this software may not  be called "Apache", nor may
      "Apache" appear  in their name,  without prior written permission  of the
      Apache Software Foundation.
  
   THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
   INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
   FITNESS  FOR A PARTICULAR  PURPOSE ARE  DISCLAIMED.  IN NO  EVENT SHALL  THE
   APACHE SOFTWARE  FOUNDATION  OR ITS CONTRIBUTORS  BE LIABLE FOR  ANY DIRECT,
   INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL  DAMAGES (INCLU-
   DING, BUT NOT LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS
   OF USE, DATA, OR  PROFITS; OR BUSINESS  INTERRUPTION)  HOWEVER CAUSED AND ON
   ANY  THEORY OF LIABILITY,  WHETHER  IN CONTRACT,  STRICT LIABILITY,  OR TORT
   (INCLUDING  NEGLIGENCE OR  OTHERWISE) ARISING IN  ANY WAY OUT OF THE  USE OF
   THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
   This software  consists of voluntary contributions made  by many individuals
   on  behalf of the Apache Software  Foundation and was  originally created by
   Stefano Mazzocchi  <stefano@apache.org>. For more  information on the Apache
   Software Foundation, please see <http://www.apache.org/>.
  
  */
  package org.apache.cocoon.transformation;
  
  import org.apache.avalon.framework.activity.Disposable;
  import org.apache.avalon.framework.component.ComponentException;
  import org.apache.avalon.framework.component.ComponentManager;
  import org.apache.avalon.framework.configuration.Configurable;
  import org.apache.avalon.framework.configuration.Configuration;
  import org.apache.avalon.framework.configuration.ConfigurationException;
  import org.apache.avalon.framework.logger.AbstractLoggable;
  import org.apache.avalon.framework.context.ContextException;
  import org.apache.avalon.framework.context.Contextualizable;
  import org.apache.avalon.framework.context.Context;
  import org.apache.avalon.framework.parameters.Parameters;
  import org.apache.avalon.excalibur.pool.Recyclable;
  
  import org.apache.cocoon.ProcessingException;
  import org.apache.cocoon.Constants;
  import org.apache.cocoon.caching.Cacheable;
  import org.apache.cocoon.caching.NOPCacheValidity;
  import org.apache.cocoon.caching.CacheValidity;
  import org.apache.cocoon.environment.SourceResolver;
  import org.apache.cocoon.components.search.LuceneXMLIndexer;
  import org.apache.cocoon.components.search.LuceneCocoonHelper;
  
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.document.Field;
  import org.apache.lucene.analysis.Analyzer;
  
  import org.xml.sax.Attributes;
  import org.xml.sax.SAXException;
  import org.xml.sax.helpers.AttributesImpl;
  
  import java.io.IOException;
  import java.io.File;
  import java.util.Stack;
  import java.util.Map;
  
  /**
   * A lucene transformer.
   * <p>FIXME: Write Documentation.</p>
   *
   * @author     <a href="mailto:vgritsenko@apache.org">Vadim Gritsenko</a>
   * @version    CVS $Revision: 1.1 $ $Date: 2002/11/22 16:56:34 $
   */
  public class LuceneTransformer extends AbstractTransformer
      implements Disposable, Cacheable, Recyclable, Configurable, Contextualizable {
  
      public static final String ANALYZER_CLASSNAME_CONFIG = "analyzer-classname";
      public static final String ANALYZER_CLASSNAME_PARAMETER = "analyzer-classname";
      public static final String ANALYZER_CLASSNAME_DEFAULT = "org.apache.lucene.analysis.standard.StandardAnalyzer";
      public static final String DIRECTORY_CONFIG = "directory";
      public static final String DIRECTORY_PARAMETER = "directory";
      public static final String DIRECTORY_DEFAULT = "index";
      public static final String MERGE_FACTOR_CONFIG = "merge-factor";
      public static final String MERGE_FACTOR_PARAMETER = "merge-factor";
      public static final int MERGE_FACTOR_DEFAULT = 20;
  
      public static final String LUCENE_URI = "http://apache.org/cocoon/lucene/1.0";
      public static final String LUCENE_QUERY_ELEMENT = "index";
      public static final String LUCENE_QUERY_ANALYZER_ATTRIBUTE = "analyzer";
      public static final String LUCENE_QUERY_DIRECTORY_ATTRIBUTE = "directory";
      public static final String LUCENE_QUERY_CREATE_ATTRIBUTE = "create";
      public static final String LUCENE_QUERY_MERGE_FACTOR_ATTRIBUTE = "merge-factor";
      public static final String LUCENE_DOCUMENT_ELEMENT = "document";
      public static final String LUCENE_DOCUMENT_URL_ATTRIBUTE = "url";
      public static final String LUCENE_ELEMENT_ATTR_TO_TEXT_ATTRIBUTE = "text-attr";
      public static final String LUCENE_ELEMENT_ATTR_STORE_VALUE = "store";
  
      // Initialization time variables
      protected ComponentManager manager = null;
      protected File workDir = null;
  
      // Declaration time parameters values
      private String analyzerClassnameDefault;
      private String directoryDefault;
      private int mergeFactorDefault;
  
      // Invocation time parameters values
      private String analyzerClassname;
      private String directory;
      private int mergeFactor;
  
  
      // Runtime variables
      private int processing;
      private IndexWriter writer;
      private StringBuffer bodyText;
      private Document bodyDocument;
      private String bodyDocumentURL;
      private Stack elementStack = new Stack();
  
  
      private static String uid(String url) {
          return url.replace('/', '\u0000'); // + "\u0000" + DateField.timeToString(urlConnection.getLastModified());
      }
  
  
      public void configure(Configuration conf) throws ConfigurationException {
          this.analyzerClassnameDefault = conf.getChild(ANALYZER_CLASSNAME_CONFIG)
              .getValue(ANALYZER_CLASSNAME_DEFAULT);
          this.mergeFactorDefault = conf.getChild(MERGE_FACTOR_CONFIG)
              .getValueAsInteger(MERGE_FACTOR_DEFAULT);
          this.directoryDefault = conf.getChild(DIRECTORY_CONFIG)
              .getValue(DIRECTORY_DEFAULT);
      }
  
      /**
       * Setup the transformer.
       */
      public void setup(SourceResolver resolver, Map objectModel, String src, Parameters parameters)
      throws ProcessingException, SAXException, IOException {
          // We don't need all this stuff
          this.analyzerClassname = parameters.getParameter(ANALYZER_CLASSNAME_PARAMETER, analyzerClassnameDefault);
          this.directory = parameters.getParameter(DIRECTORY_PARAMETER, directoryDefault);
          this.mergeFactor = parameters.getParameterAsInteger(MERGE_FACTOR_PARAMETER, mergeFactorDefault);
      }
  
      public void compose(ComponentManager manager) throws ComponentException {
          this.manager = manager;
      }
  
      /**
       * Contextualize this class
       */
      public void contextualize(Context context) throws ContextException {
          this.workDir = (File) context.get(Constants.CONTEXT_WORK_DIR);
      }
  
      public void recycle() {
          this.processing = 0;
          if (this.writer != null) {
              try { this.writer.close(); } catch (IOException ioe) { }
              this.writer = null;
          }
          this.bodyText = null;
          this.bodyDocument = null;
          this.bodyDocumentURL = null;
          this.elementStack.clear();
      }
  
      public void dispose() {
      }
  
      /**
       * Generate the unique key.
       * This key must be unique inside the space of this component.
       *
       * @return The generated key hashes the src
       */
      public long generateKey() {
          return 1;
      }
  
      /**
       * Generate the validity object.
       *
       * @return The generated validity object or <code>null</code> if the
       *         component is currently not cacheable.
       */
      public CacheValidity generateValidity() {
          return NOPCacheValidity.CACHE_VALIDITY;
      }
  
  
  
      public void startDocument() throws SAXException {
          super.startDocument();
      }
  
      public void endDocument() throws SAXException {
          super.endDocument();
      }
  
      /**
       * Begin the scope of a prefix-URI Namespace mapping.
       *
       * @param prefix The Namespace prefix being declared.
       * @param uri The Namespace URI the prefix is mapped to.
       */
      public void startPrefixMapping(String prefix, String uri) throws SAXException {
          if (processing == 0) {
              super.startPrefixMapping(prefix,uri);
          }
      }
  
      /**
       * End the scope of a prefix-URI mapping.
       *
       * @param prefix The prefix that was being mapping.
       */
      public void endPrefixMapping(String prefix) throws SAXException {
          if (processing == 0) {
              super.endPrefixMapping(prefix);
          }
      }
  
      public void startElement(String namespaceURI, String localName, String qName, Attributes
atts)
          throws SAXException {
  
          if (processing == 0) {
              if (LUCENE_URI.equals(namespaceURI) && LUCENE_QUERY_ELEMENT.equals(localName)){
                  String sCreate = atts.getValue(LUCENE_QUERY_CREATE_ATTRIBUTE);
                  boolean bCreate = sCreate != null &&
                      (sCreate.equalsIgnoreCase("yes") || sCreate.equalsIgnoreCase("true"));
  
                  String analyzerClassname =
                      atts.getValue(LUCENE_QUERY_ANALYZER_ATTRIBUTE);
                  if (analyzerClassname == null)
                      analyzerClassname = this.analyzerClassname;
                  Analyzer analyzer = LuceneCocoonHelper.getAnalyzer(analyzerClassname);
  
                  String sMergeFactor =
                      atts.getValue(LUCENE_QUERY_MERGE_FACTOR_ATTRIBUTE);
                  int mergeFactor = this.mergeFactor;
                  if (sMergeFactor != null)
                      mergeFactor = Integer.parseInt(sMergeFactor);
  
                  String directoryName =
                      atts.getValue(LUCENE_QUERY_DIRECTORY_ATTRIBUTE);
                  if (directoryName == null)
                      directoryName = this.directory;
  
                  // System.out.println("QUERY Create=" + bCreate + ", Directory=" + directoryName
+ ", Analyzer=" + analyzerClassname);
                  try {
                      Directory directory = LuceneCocoonHelper.getDirectory(
                          new File(workDir, directoryName), bCreate);
  
                      writer = new IndexWriter(directory, analyzer, bCreate);
                      writer.mergeFactor = mergeFactor;
                  } catch (IOException e) {
                      throw new SAXException(e);
                  }
  
                  processing = 1;
              } else {
                  super.startElement(namespaceURI, localName, qName, atts);
              }
          } else if (processing == 1) {
              if (LUCENE_URI.equals(namespaceURI) && LUCENE_DOCUMENT_ELEMENT.equals(localName)){
                  this.bodyDocumentURL = atts.getValue(LUCENE_DOCUMENT_URL_ATTRIBUTE);
                  if (this.bodyDocumentURL == null)
                      throw new SAXException("<lucene:document> must have @url attribute");
  
                  // System.out.println("  DOCUMENT URL=" + bodyDocumentURL);
                  this.bodyText = new StringBuffer();
                  this.bodyDocument = new Document();
                  this.elementStack.clear();
                  processing = 2;
              } else {
                  throw new SAXException("<lucene:query> element can contain only <lucene:document>
elements!");
              }
          } else if (processing == 2) {
              elementStack.push(new IndexHelperField(localName, new AttributesImpl(atts)));
          }
      }
  
      public void endElement(String namespaceURI, String localName, String qName)
          throws SAXException {
  
          if (processing == 1) {
              if (LUCENE_URI.equals(namespaceURI) && LUCENE_QUERY_ELEMENT.equals(localName))
{
                  // End query processing
                  // System.out.println("QUERY END!");
                  try {
                      this.writer.optimize();
                      this.writer.close();
                      this.writer = null;
                  } catch (IOException e) {
                      throw new SAXException(e);
                  }
  
                  this.processing = 0;
              } else {
                  throw new SAXException("</lucene:query> was expected!");
              }
          } else if (processing == 2) {
              if (LUCENE_URI.equals(namespaceURI) && LUCENE_DOCUMENT_ELEMENT.equals(localName))
{
                  // End document processing
                  this.bodyDocument.add(Field.UnStored(LuceneXMLIndexer.BODY_FIELD, this.bodyText.toString()));
                  System.out.println("    DOCUMENT BODY=" + this.bodyText);
                  this.bodyText = null;
  
                  this.bodyDocument.add(Field.UnIndexed(LuceneXMLIndexer.URL_FIELD, this.bodyDocumentURL));
                  // store: false, index: true, tokenize: false
                  this.bodyDocument.add(new Field(LuceneXMLIndexer.UID_FIELD, uid(this.bodyDocumentURL),
false, true, false));
                  // System.out.println("    DOCUMENT UID=" + uid(this.bodyDocumentURL));
                  this.bodyDocumentURL = null;
                  // System.out.println("  DOCUMENT END!");
                  try {
                      this.writer.addDocument(this.bodyDocument);
                      this.bodyDocument = null;
                  } catch (IOException e) {
                      throw new SAXException(e);
                  }
  
                  this.processing = 1;
              } else {
                  // End element processing
                  IndexHelperField tos = (IndexHelperField) elementStack.pop();
                  StringBuffer text = tos.getText();
  
                  Attributes atts = tos.getAttributes();
                  boolean attributesToText = atts.getIndex(LUCENE_URI, LUCENE_ELEMENT_ATTR_TO_TEXT_ATTRIBUTE)
!= -1;
                  for (int i = 0; atts != null && i < atts.getLength(); i++) {
                      // Ignore Lucene attributes
                      if (LUCENE_URI.equals(atts.getURI(i)))
                          continue;
  
                      String atts_lname = atts.getLocalName(i);
                      String atts_value = atts.getValue(i);
                      // System.out.println("        ATTRIBUTE " + localName + "@" + atts_lname
+ "=" + atts_value);
                      bodyDocument.add(Field.UnStored(localName + "@" + atts_lname, atts_value));
                      if (attributesToText) {
                          text.append(atts_value);
                          text.append(' ');
                          bodyText.append(atts_value);
                          bodyText.append(' ');
                      }
                  }
  
                  boolean store = atts.getIndex(LUCENE_URI, LUCENE_ELEMENT_ATTR_STORE_VALUE)
!= -1;
                  if (text != null && text.length() > 0) {
                      // System.out.println("      ELEMENT " + localName + "=" + text);
                      if (store) {
                          bodyDocument.add(Field.Text(localName, text.toString()));
                      } else {
                          bodyDocument.add(Field.UnStored(localName, text.toString()));
                      }
                  }
              }
          } else {
              // All other tags
              super.endElement(namespaceURI, localName, qName);
          }
      }
  
      public void characters(char[] ch, int start, int length)
          throws SAXException {
  
          if (processing == 2 && ch.length > 0 && start >= 0 &&
length > 1 && elementStack.size() > 0) {
              String text = new String(ch, start, length);
              ((IndexHelperField) elementStack.peek()).append(text);
              bodyText.append(text);
              bodyText.append(' ');
          } else if (processing == 0) {
              super.characters(ch, start, length);
          }
      }
  
  
      class IndexHelperField
      {
          String localName;
          StringBuffer text;
          Attributes attributes;
  
          IndexHelperField(String localName, Attributes atts) {
              this.localName = localName;
              this.attributes = atts;
              this.text = new StringBuffer();
          }
  
          public Attributes getAttributes() {
              return attributes;
          }
  
          public StringBuffer getText() {
              return text;
          }
  
          public void append(String text) {
              this.text.append(text);
          }
  
          public void append(char[] str, int offset, int length) {
              this.text.append(str, offset, length);
          }
      }
  }
  
  
  

----------------------------------------------------------------------
In case of troubles, e-mail:     webmaster@xml.apache.org
To unsubscribe, e-mail:          cocoon-cvs-unsubscribe@xml.apache.org
For additional commands, e-mail: cocoon-cvs-help@xml.apache.org


Mime
View raw message