cocoon-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vgritse...@apache.org
Subject cvs commit: xml-cocoon2/src/java/org/apache/cocoon/components/crawler SimpleCocoonCrawlerImpl.java
Date Sat, 17 Aug 2002 04:04:12 GMT
vgritsenko    2002/08/16 21:04:12

  Modified:    src/java/org/apache/cocoon/components/crawler
                        SimpleCocoonCrawlerImpl.java
  Log:
  reformat
  
  Revision  Changes    Path
  1.15      +88 -88    xml-cocoon2/src/java/org/apache/cocoon/components/crawler/SimpleCocoonCrawlerImpl.java
  
  Index: SimpleCocoonCrawlerImpl.java
  ===================================================================
  RCS file: /home/cvs/xml-cocoon2/src/java/org/apache/cocoon/components/crawler/SimpleCocoonCrawlerImpl.java,v
  retrieving revision 1.14
  retrieving revision 1.15
  diff -u -r1.14 -r1.15
  --- SimpleCocoonCrawlerImpl.java	7 Aug 2002 10:52:13 -0000	1.14
  +++ SimpleCocoonCrawlerImpl.java	17 Aug 2002 04:04:11 -0000	1.15
  @@ -1,36 +1,36 @@
   /*
  - 
  +
    ============================================================================
                      The Apache Software License, Version 1.1
    ============================================================================
  - 
  +
    Copyright (C) 1999-2002 The Apache Software Foundation. All rights reserved.
  - 
  +
    Redistribution and use in source and binary forms, with or without modifica-
    tion, are permitted provided that the following conditions are met:
  - 
  +
    1. Redistributions of  source code must  retain the above copyright  notice,
       this list of conditions and the following disclaimer.
  - 
  +
    2. Redistributions in binary form must reproduce the above copyright notice,
       this list of conditions and the following disclaimer in the documentation
       and/or other materials provided with the distribution.
  - 
  +
    3. The end-user documentation included with the redistribution, if any, must
       include  the following  acknowledgment:  "This product includes  software
       developed  by the  Apache Software Foundation  (http://www.apache.org/)."
       Alternately, this  acknowledgment may  appear in the software itself,  if
       and wherever such third-party acknowledgments normally appear.
  - 
  +
    4. The names "Apache Cocoon" and  "Apache Software Foundation" must  not  be
       used to  endorse or promote  products derived from  this software without
       prior written permission. For written permission, please contact
       apache@apache.org.
  - 
  +
    5. Products  derived from this software may not  be called "Apache", nor may
       "Apache" appear  in their name,  without prior written permission  of the
       Apache Software Foundation.
  - 
  +
    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    FITNESS  FOR A PARTICULAR  PURPOSE ARE  DISCLAIMED.  IN NO  EVENT SHALL  THE
  @@ -41,12 +41,12 @@
    ANY  THEORY OF LIABILITY,  WHETHER  IN CONTRACT,  STRICT LIABILITY,  OR TORT
    (INCLUDING  NEGLIGENCE OR  OTHERWISE) ARISING IN  ANY WAY OUT OF THE  USE OF
    THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  - 
  +
    This software  consists of voluntary contributions made  by many individuals
    on  behalf of the Apache Software  Foundation and was  originally created by
    Stefano Mazzocchi  <stefano@apache.org>. For more  information on the Apache
    Software Foundation, please see <http://www.apache.org/>.
  - 
  +
    */
   package org.apache.cocoon.components.crawler;
   
  @@ -84,8 +84,7 @@
    * @version CVS $Id$
    */
   public class SimpleCocoonCrawlerImpl extends AbstractLogEnabled
  -    implements CocoonCrawler, Configurable, Disposable, Recyclable
  -{
  +        implements CocoonCrawler, Configurable, Disposable, Recyclable {
   
       /**
        * Config element name specifying expected link content-typ.
  @@ -96,7 +95,7 @@
        * @since
        */
       public final static String LINK_CONTENT_TYPE_CONFIG = "link-content-type";
  -    
  +
       /**
        * Default value of <code>link-content-type</code> configuration value.
        * <p>
  @@ -106,7 +105,7 @@
        * @since
        */
       public final String LINK_CONTENT_TYPE_DEFAULT = "application/x-cocoon-links";
  -    
  +
       /**
        * Config element name specifying query-string appendend for requesting links
        * of an URL.
  @@ -117,7 +116,7 @@
        * @since
        */
       public final static String LINK_VIEW_QUERY_CONFIG = "link-view-query";
  -    
  +
       /**
        * Default value of <code>link-view-query</code> configuration value.
        * <p>
  @@ -127,7 +126,7 @@
        * @since
        */
       public final static String LINK_VIEW_QUERY_DEFAULT = "cocoon-view=links";
  -    
  +
       /**
        * Config element name specifying excluding regular expression pattern.
        * <p>
  @@ -137,7 +136,7 @@
        * @since
        */
       public final static String EXCLUDE_CONFIG = "exclude";
  -    
  +
       /**
        * Config element name specifying including regular expression pattern.
        * <p>
  @@ -147,7 +146,7 @@
        * @since
        */
       public final static String INCLUDE_CONFIG = "include";
  -    
  +
       /**
        * Config element name specifying http header value for user-Agent.
        * <p>
  @@ -157,7 +156,7 @@
        * @since
        */
       public final static String USER_AGENT_CONFIG = "user-agent";
  -    
  +
       /**
        * Default value of <code>user-agent</code> configuration value.
        * @see Constants#COMPLETE_NAME
  @@ -165,7 +164,7 @@
        * @since
        */
       public final static String USER_AGENT_DEFAULT = Constants.COMPLETE_NAME;
  -    
  +
       /**
        * Config element name specifying http header value for accept.
        * <p>
  @@ -175,7 +174,7 @@
        * @since
        */
       public final static String ACCEPT_CONFIG = "accept";
  -    
  +
       /**
        * Default value of <code>accept</code> configuration value.
        * <p>
  @@ -185,18 +184,18 @@
        * @since
        */
       public final static String ACCEPT_DEFAULT = "*/*";
  -    
  +
       private String linkViewQuery = LINK_VIEW_QUERY_DEFAULT;
       private String linkContentType = LINK_CONTENT_TYPE_DEFAULT;
       private HashSet excludeCrawlingURL;
       private HashSet includeCrawlingURL;
       private String userAgent = USER_AGENT_DEFAULT;
       private String accept = ACCEPT_DEFAULT;
  -    
  +
       private HashSet crawled;
       private HashSet urlsToProcess;
  -    
  -    
  +
  +
       /**
        * Constructor for the SimpleCocoonCrawlerImpl object
        *
  @@ -208,8 +207,8 @@
           // by default exclude common image patterns
           excludeCrawlingURL = null;
       }
  -    
  -    
  +
  +
       /**
        * Configure the crawler component.
        * <p>
  @@ -233,8 +232,8 @@
        * @since
        */
       public void configure(Configuration configuration)
  -    throws ConfigurationException {
  -        
  +            throws ConfigurationException {
  +
           Configuration[] children;
           children = configuration.getChildren(INCLUDE_CONFIG);
           if (children != null && children.length > 0) {
  @@ -249,7 +248,7 @@
                       }
                   } catch (RESyntaxException rese) {
                       getLogger().error("Cannot create including regular-expression for "
+
  -                    pattern, rese);
  +                            pattern, rese);
                   }
               }
           } else {
  @@ -257,7 +256,7 @@
                   getLogger().debug("Include all URLs");
               }
           }
  -        
  +
           children = configuration.getChildren(EXCLUDE_CONFIG);
           if (children != null && children.length > 0) {
               excludeCrawlingURL = new HashSet();
  @@ -271,7 +270,7 @@
                       }
                   } catch (RESyntaxException rese) {
                       getLogger().error("Cannot create excluding regular-expression for "
+
  -                    pattern, rese);
  +                            pattern, rese);
                   }
               }
           } else {
  @@ -281,7 +280,7 @@
                   getLogger().debug("Exclude default URLs only");
               }
           }
  -        
  +
           Configuration child;
           String value;
           child = configuration.getChild(LINK_CONTENT_TYPE_CONFIG, false);
  @@ -298,7 +297,7 @@
                   this.linkViewQuery = value.trim();
               }
           }
  -        
  +
           child = configuration.getChild(USER_AGENT_CONFIG, false);
           if (child != null) {
               value = child.getValue();
  @@ -306,7 +305,7 @@
                   this.userAgent = value;
               }
           }
  -        
  +
           child = configuration.getChild(ACCEPT_CONFIG, false);
           if (child != null) {
               value = child.getValue();
  @@ -315,8 +314,8 @@
               }
           }
       }
  -    
  -    
  +
  +
       /**
        * dispose at end of life cycle, releasing all resources.
        *
  @@ -328,8 +327,8 @@
           excludeCrawlingURL = null;
           includeCrawlingURL = null;
       }
  -    
  -    
  +
  +
       /**
        * recylcle this object, relasing resources
        *
  @@ -339,8 +338,8 @@
           crawled = null;
           urlsToProcess = null;
       }
  -    
  -    
  +
  +
       /**
        * Start crawling a URL.
        *
  @@ -372,15 +371,15 @@
       public void crawl(URL url) {
           crawled = new HashSet();
           urlsToProcess = new HashSet();
  -        
  +
           if (getLogger().isDebugEnabled()) {
               getLogger().debug("crawl URL " + url);
           }
  -        
  +
           urlsToProcess.add(url);
       }
  -    
  -    
  +
  +
       /**
        * Return iterator, iterating over all links of the currently crawled URL.
        * <p>
  @@ -394,8 +393,8 @@
       public Iterator iterator() {
           return new CocoonCrawlerIterator(this);
       }
  -    
  -    
  +
  +
       /**
        * Default exclude patterns.
        * <p>
  @@ -419,19 +418,19 @@
               ".*\\.js(\\?.*)?$",
               ".*\\.css(\\?.*)?$"
           };
  -        
  +
           for (int i = 0; i < EXCLUDE_FROM_CRAWLING_DEFAULT.length; i++) {
               String pattern = EXCLUDE_FROM_CRAWLING_DEFAULT[i];
               try {
                   excludeCrawlingURL.add(new RE(pattern));
               } catch (RESyntaxException rese) {
                   getLogger().error("Cannot create excluding regular-expression for " +
  -                pattern, rese);
  +                        pattern, rese);
               }
           }
       }
  -    
  -    
  +
  +
       /**
        * Compute list of links from the url.
        * <p>
  @@ -446,19 +445,19 @@
       private List getLinks(URL url) {
           ArrayList url_links = null;
           String sURL = url.toString();
  -        
  +
           if (!isIncludedURL(sURL) || isExcludedURL(sURL)) {
               return null;
           }
  -        
  +
           // don't try to get links for url which has been crawled already
           if (crawled.contains(sURL)) {
               return null;
           }
  -        
  +
           // mark it as crawled
           crawled.add(sURL);
  -        
  +
           // get links of url
           if (getLogger().isDebugEnabled()) {
               getLogger().debug("Getting links of URL " + sURL);
  @@ -467,12 +466,12 @@
           try {
               sURL = url.getFile();
               URL links = new URL(url, sURL
  -            + ((sURL.indexOf("?") == -1) ? "?" : "&")
  -            + linkViewQuery);
  +                    + ((sURL.indexOf("?") == -1) ? "?" : "&")
  +                    + linkViewQuery);
               URLConnection links_url_connection = links.openConnection();
               InputStream is = links_url_connection.getInputStream();
               br = new BufferedReader(new InputStreamReader(is));
  -            
  +
               String contentType = links_url_connection.getContentType();
               if (contentType == null) {
                   if (getLogger().isDebugEnabled()) {
  @@ -481,19 +480,19 @@
                   // there is a check on null in the calling method
                   return null;
               }
  -            
  +
               int index = contentType.indexOf(';');
               if (index != -1) {
                   contentType = contentType.substring(0, index);
               }
  -            
  +
               if (getLogger().isDebugEnabled()) {
                   getLogger().debug("Content-type: " + contentType);
               }
  -            
  +
               if (contentType.equals(linkContentType)) {
                   url_links = new ArrayList();
  -                
  +
                   // content is supposed to be a list of links,
                   // relative to current URL
                   String line;
  @@ -504,17 +503,17 @@
                       if (add_url) {
                           add_url &= !url_links.contains(new_url);
                       }
  -                    
  +
                       // don't add new_url if it has been crawled already
                       if (add_url) {
                           add_url &= !crawled.contains(new_url.toString());
                       }
  -                    
  +
                       // don't add if is not matched by existing include definition
                       if (add_url) {
                           add_url &= isIncludedURL(new_url.toString());
                       }
  -                    
  +
                       // don't add if is matched by existing exclude definition
                       if (add_url) {
                           add_url &= !isExcludedURL(new_url.toString());
  @@ -531,17 +530,18 @@
           } catch (IOException ioe) {
               getLogger().warn("Problems get links of " + url, ioe);
           } finally {
  -            if( br != null ) {
  +            if (br != null) {
                   try {
                       br.close();
                       br = null;
  -                } catch (IOException ignored ) {}
  +                } catch (IOException ignored) {
  +                }
               }
           }
           return url_links;
       }
  -    
  -    
  +
  +
       /**
        * check if URL is a candidate for indexing
        *
  @@ -554,7 +554,7 @@
           if (excludeCrawlingURL == null) {
               return false;
           }
  -        
  +
           final String s = url.toString();
           Iterator i = excludeCrawlingURL.iterator();
           while (i.hasNext()) {
  @@ -571,8 +571,8 @@
           }
           return false;
       }
  -    
  -    
  +
  +
       /**
        * check if URL is a candidate for indexing
        *
  @@ -585,7 +585,7 @@
           if (includeCrawlingURL == null) {
               return true;
           }
  -        
  +
           final String s = url.toString();
           Iterator i = includeCrawlingURL.iterator();
           while (i.hasNext()) {
  @@ -602,8 +602,8 @@
           }
           return false;
       }
  -    
  -    
  +
  +
       /**
        * Helper class implementing an Iterator
        * <p>
  @@ -616,8 +616,8 @@
        */
       public static class CocoonCrawlerIterator implements Iterator {
           private SimpleCocoonCrawlerImpl cocoonCrawler;
  -        
  -        
  +
  +
           /**
            *Constructor for the CocoonCrawlerIterator object
            *
  @@ -627,8 +627,8 @@
           CocoonCrawlerIterator(SimpleCocoonCrawlerImpl cocoonCrawler) {
               this.cocoonCrawler = cocoonCrawler;
           }
  -        
  -        
  +
  +
           /**
            * check if crawling is finished.
            *
  @@ -638,8 +638,8 @@
           public boolean hasNext() {
               return cocoonCrawler.urlsToProcess.size() > 0;
           }
  -        
  -        
  +
  +
           /**
            * return the next URL
            *
  @@ -652,10 +652,10 @@
               if (i.hasNext()) {
                   // fetch a URL
                   url = (URL) i.next();
  -                
  +
                   // remove it from the to-do list
                   cocoonCrawler.urlsToProcess.remove(url);
  -                
  +
                   // calc all links from this url
                   List url_links = cocoonCrawler.getLinks(url);
                   if (url_links != null) {
  @@ -666,8 +666,8 @@
               // finally return this url
               return url;
           }
  -        
  -        
  +
  +
           /**
            * remove is not implemented
            *
  
  
  

----------------------------------------------------------------------
In case of troubles, e-mail:     webmaster@xml.apache.org
To unsubscribe, e-mail:          cocoon-cvs-unsubscribe@xml.apache.org
For additional commands, e-mail: cocoon-cvs-help@xml.apache.org


Mime
View raw message