cocoon-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vgritse...@apache.org
Subject cvs commit: xml-cocoon2/src/java/org/apache/cocoon/generation LinkStatusGenerator.java
Date Sat, 17 Aug 2002 03:25:39 GMT
vgritsenko    2002/08/16 20:25:39

  Modified:    src/java/org/apache/cocoon/generation
                        LinkStatusGenerator.java
  Log:
  Formatting changes
  
  Revision  Changes    Path
  1.6       +88 -88    xml-cocoon2/src/java/org/apache/cocoon/generation/LinkStatusGenerator.java
  
  Index: LinkStatusGenerator.java
  ===================================================================
  RCS file: /home/cvs/xml-cocoon2/src/java/org/apache/cocoon/generation/LinkStatusGenerator.java,v
  retrieving revision 1.5
  retrieving revision 1.6
  diff -u -r1.5 -r1.6
  --- LinkStatusGenerator.java	4 Aug 2002 18:33:51 -0000	1.5
  +++ LinkStatusGenerator.java	17 Aug 2002 03:25:39 -0000	1.6
  @@ -1,36 +1,36 @@
   /*
  - 
  +
    ============================================================================
                      The Apache Software License, Version 1.1
    ============================================================================
  - 
  +
    Copyright (C) 1999-2002 The Apache Software Foundation. All rights reserved.
  - 
  +
    Redistribution and use in source and binary forms, with or without modifica-
    tion, are permitted provided that the following conditions are met:
  - 
  +
    1. Redistributions of  source code must  retain the above copyright  notice,
       this list of conditions and the following disclaimer.
  - 
  +
    2. Redistributions in binary form must reproduce the above copyright notice,
       this list of conditions and the following disclaimer in the documentation
       and/or other materials provided with the distribution.
  - 
  +
    3. The end-user documentation included with the redistribution, if any, must
       include  the following  acknowledgment:  "This product includes  software
       developed  by the  Apache Software Foundation  (http://www.apache.org/)."
       Alternately, this  acknowledgment may  appear in the software itself,  if
       and wherever such third-party acknowledgments normally appear.
  - 
  +
    4. The names "Apache Cocoon" and  "Apache Software Foundation" must  not  be
       used to  endorse or promote  products derived from  this software without
       prior written permission. For written permission, please contact
       apache@apache.org.
  - 
  +
    5. Products  derived from this software may not  be called "Apache", nor may
       "Apache" appear  in their name,  without prior written permission  of the
       Apache Software Foundation.
  - 
  +
    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    FITNESS  FOR A PARTICULAR  PURPOSE ARE  DISCLAIMED.  IN NO  EVENT SHALL  THE
  @@ -41,12 +41,12 @@
    ANY  THEORY OF LIABILITY,  WHETHER  IN CONTRACT,  STRICT LIABILITY,  OR TORT
    (INCLUDING  NEGLIGENCE OR  OTHERWISE) ARISING IN  ANY WAY OUT OF THE  USE OF
    THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  - 
  +
    This software  consists of voluntary contributions made  by many individuals
    on  behalf of the Apache Software  Foundation and was  originally created by
    Stefano Mazzocchi  <stefano@apache.org>. For more  information on the Apache
    Software Foundation, please see <http://www.apache.org/>.
  - 
  +
    */
   package org.apache.cocoon.generation;
   
  @@ -91,22 +91,22 @@
       /** The URI of the namespace of this generator. */
       protected static final String URI =
       "http://apache.org/cocoon/linkstatus/2.0";
  -    
  +
       /** The namespace prefix for this namespace. */
       protected static final String PREFIX = "linkstatus";
  -    
  +
       /* Node and attribute names */
       protected static final String TOP_NODE_NAME         = "linkstatus";
       protected static final String LINK_NODE_NAME         = "link";
  -    
  +
       protected static final String HREF_ATTR_NAME    = "href";
       protected static final String REFERRER_ATTR_NAME    = "referrer";
       protected static final String CONTENT_ATTR_NAME    = "content";
       protected static final String STATUS_ATTR_NAME    = "status";
       protected static final String MESSAGE_ATTR_NAME    = "message";
  -    
  +
       protected AttributesImpl attributes = new AttributesImpl();
  -    
  +
       /**
        * Config element name specifying expected link content-typ.
        * <p>
  @@ -116,7 +116,7 @@
        * @since
        */
       public final static String LINK_CONTENT_TYPE_CONFIG = "link-content-type";
  -    
  +
       /**
        * Default value of <code>link-content-type</code> configuration value.
        * <p>
  @@ -126,7 +126,7 @@
        * @since
        */
       public final String LINK_CONTENT_TYPE_DEFAULT = "application/x-cocoon-links";
  -    
  +
       /**
        * Config element name specifying query-string appendend for requesting links
        * of an URL.
  @@ -146,7 +146,7 @@
        * @since
        */
       public final static String LINK_VIEW_QUERY_DEFAULT = "cocoon-view=links";
  -    
  +
       /**
        * Config element name specifying excluding regular expression pattern.
        * <p>
  @@ -156,7 +156,7 @@
        * @since
        */
       public final static String EXCLUDE_CONFIG = "exclude";
  -    
  +
       /**
        * Config element name specifying including regular expression pattern.
        * <p>
  @@ -166,7 +166,7 @@
        * @since
        */
       public final static String INCLUDE_CONFIG = "include";
  -    
  +
       /**
        * Config element name specifying http header value for user-Agent.
        * <p>
  @@ -183,7 +183,7 @@
        * @since
        */
       public final static String USER_AGENT_DEFAULT = Constants.COMPLETE_NAME;
  -    
  +
       /**
        * Config element name specifying http header value for accept.
        * <p>
  @@ -202,42 +202,42 @@
        * @since
        */
       public final static String ACCEPT_DEFAULT = "*/*";
  -    
  +
       private String linkViewQuery = LINK_VIEW_QUERY_DEFAULT;
       private String linkContentType = LINK_CONTENT_TYPE_DEFAULT;
       private HashSet excludeCrawlingURL;
       private HashSet includeCrawlingURL;
       private String userAgent = USER_AGENT_DEFAULT;
       private String accept = ACCEPT_DEFAULT;
  -    
  +
       private HashSet crawled;
       private HashSet linksToProcess;
  -    
  +
       /**
        * Stores links to process and the referrer links
        */
       private class Link {
           private URL url;
           private String referrer;
  -        
  +
           public Link( URL url, String referrer ) {
               this.url = url;
               this.referrer = referrer;
           }
  -        
  +
           public URL getURL() {
               return url;
           }
  -        
  +
           public String getReferrer() {
               return referrer;
           }
  -        
  +
           public boolean equals( Link l ) {
               return url.equals( l.getURL());
           }
       }
  -    
  +
       /**
        * Configure the crawler component.
        * <p>
  @@ -264,7 +264,7 @@
        */
       public void configure(Configuration configuration)
       throws ConfigurationException {
  -        
  +
           Configuration[] children;
           children = configuration.getChildren(INCLUDE_CONFIG);
           if (children != null && children.length > 0) {
  @@ -283,7 +283,7 @@
                   }
               }
           }
  -        
  +
           children = configuration.getChildren(EXCLUDE_CONFIG);
           if (children != null && children.length > 0) {
               excludeCrawlingURL = new HashSet();
  @@ -304,7 +304,7 @@
               excludeCrawlingURL = new HashSet();
               setDefaultExcludeFromCrawling();
           }
  -        
  +
           Configuration child;
           String value;
           child = configuration.getChild(LINK_CONTENT_TYPE_CONFIG, false);
  @@ -321,7 +321,7 @@
                   this.linkViewQuery = value.trim();
               }
           }
  -        
  +
           child = configuration.getChild(USER_AGENT_CONFIG, false);
           if (child != null) {
               value = child.getValue();
  @@ -329,7 +329,7 @@
                   this.userAgent = value;
               }
           }
  -        
  +
           child = configuration.getChild(ACCEPT_CONFIG, false);
           if (child != null) {
               value = child.getValue();
  @@ -338,20 +338,20 @@
               }
           }
       }
  -    
  +
       public void setup(SourceResolver resolver, Map objectModel, String src, Parameters
par)
       throws ProcessingException, SAXException, IOException {
  -        
  +
           super.setup(resolver, objectModel, src, par);
  -        
  +
           /* Create a reusable attributes for creating nodes */
           this.attributes = new AttributesImpl();
  -        
  +
           // already done in configure...
           //excludeCrawlingURL = new HashSet();
           //this.setDefaultExcludeFromCrawling();
       }
  -    
  +
       /**
        * Generate XML data.
        *
  @@ -363,40 +363,40 @@
       public void generate()
       throws SAXException, ProcessingException {
           try {
  -            
  +
               crawled = new HashSet();
               linksToProcess = new HashSet();
  -            
  +
               URL root = new URL(source);
               linksToProcess.add(new Link( root, ""));
  -            
  -            
  +
  +
               if (getLogger().isDebugEnabled()) {
                   getLogger().debug("crawl URL " + root);
               }
  -            
  +
               this.contentHandler.startDocument();
               this.contentHandler.startPrefixMapping(PREFIX,URI);
  -            
  +
               attributes.clear();
               super.contentHandler.startElement(URI, TOP_NODE_NAME, URI+':'+TOP_NODE_NAME,
attributes);
  -            
  +
               while (linksToProcess.size() > 0) {
                   Iterator i = linksToProcess.iterator();
  -                
  +
                   if (i.hasNext()) {
                       // fetch a URL
                       Link link = (Link) i.next();
                       URL url = link.getURL();
  -                    
  +
                       // remove it from the to-do list
                       linksToProcess.remove(link);
  -                    
  +
                       String new_url_link = processURL(url, link.getReferrer());
  -                    
  +
                       // calc all links from this url
                       if (new_url_link != null) {
  -                        
  +
                           List url_links = getLinksFromConnection(new_url_link, url);
                           if (url_links != null) {
                               // add links of this url to the to-do list
  @@ -405,7 +405,7 @@
                       }
                   }
               }
  -            
  +
               super.contentHandler.endElement(URI, TOP_NODE_NAME, URI+':'+TOP_NODE_NAME);
               this.contentHandler.endPrefixMapping(PREFIX);
               this.contentHandler.endDocument();
  @@ -414,7 +414,7 @@
               throw new ResourceNotFoundException("Could not read source ", ioe);
           }
       }
  -    
  +
       /**
        * Default exclude patterns.
        * <p>
  @@ -438,7 +438,7 @@
               ".*\\.js(\\?.*)?$",
               ".*\\.css(\\?.*)?$"
           };
  -        
  +
           for (int i = 0; i < EXCLUDE_FROM_CRAWLING_DEFAULT.length; i++) {
               String pattern = EXCLUDE_FROM_CRAWLING_DEFAULT[i];
               try {
  @@ -449,8 +449,8 @@
               }
           }
       }
  -    
  -    
  +
  +
       /**
        * Retrieve a list of links of a url
        *
  @@ -469,28 +469,28 @@
               URL url_link = new URL( url_link_string );
               URLConnection conn = url_link.openConnection();
               String content_type = conn.getContentType();
  -            
  +
               if (content_type == null) {
                   getLogger().warn( "No content type available for " + String.valueOf( url_link_string
) );
                   // caller checks if null
                   return url_links;
               }
  -            
  +
               if (getLogger().isDebugEnabled()) {
                   getLogger().debug("Content-type: " + content_type);
               }
  -            
  +
               if (content_type.equals(linkContentType)) {
                   url_links = new ArrayList();
  -                
  +
                   InputStream is = conn.getInputStream();
                   br = new BufferedReader(new InputStreamReader(is));
  -                
  +
                   // content is supposed to be a list of links,
                   // relative to current URL
                   String line;
                   String referrer = url_of_referrer.toString();
  -                
  +
                   while ((line = br.readLine()) != null) {
                       URL new_url = new URL(url_link, line);
                       boolean add_url = true;
  @@ -498,22 +498,22 @@
                       if (add_url) {
                           add_url &= !url_links.contains(new_url);
                       }
  -                    
  +
                       // don't add new_url if it has been crawled already
                       if (add_url) {
                           add_url &= !crawled.contains(new_url.toString());
                       }
  -                    
  +
                       Link new_link = new Link( new_url, referrer );
                       if (add_url) {
                           add_url &= !linksToProcess.contains(new_link);
                       }
  -                    
  +
                       // don't add if is not matched by existing include definition
                       if (add_url) {
                           add_url &= isIncludedURL(new_url.toString());
                       }
  -                    
  +
                       if (add_url) {
                           if (getLogger().isDebugEnabled()) {
                               getLogger().debug("Add URL: " + new_url.toString());
  @@ -536,7 +536,7 @@
           }
           return url_links;
       }
  -    
  +
       /**
        * Generate xml attributes of a url, calculate url for retrieving links
        *
  @@ -546,43 +546,43 @@
        *   and not an included-url.
        */
       protected String processURL(URL url, String referrer) throws SAXException {
  -        
  +
           if (getLogger().isDebugEnabled()) {
               getLogger().debug("getLinks URL " + url);
           }
  -        
  +
           String result = null;
  -        
  +
           // don't try to investigate a url which has been crawled already
           if (crawled.contains(url.toString())) {
               return null;
           }
  -        
  +
           // mark it as crawled
           crawled.add(url.toString());
  -        
  +
           attributes.clear();
           attributes.addAttribute("", HREF_ATTR_NAME,
           HREF_ATTR_NAME, "CDATA", url.toString());
           attributes.addAttribute("", REFERRER_ATTR_NAME,
           REFERRER_ATTR_NAME, "CDATA", referrer);
  -        
  +
           // Output url, referrer, content-type, status, message for traversable url's
           HttpURLConnection h = null;
           try {
  -            
  +
               URLConnection links_url_connection = url.openConnection();
               h = (HttpURLConnection)links_url_connection;
               String content_type = links_url_connection.getContentType();
  -            
  +
               attributes.addAttribute("", CONTENT_ATTR_NAME,
               CONTENT_ATTR_NAME, "CDATA",
               content_type);
  -            
  +
               attributes.addAttribute("", MESSAGE_ATTR_NAME,
               MESSAGE_ATTR_NAME, "CDATA",
               h.getResponseMessage());
  -            
  +
               attributes.addAttribute("", STATUS_ATTR_NAME,
               STATUS_ATTR_NAME, "CDATA",
               String.valueOf(h.getResponseCode()));
  @@ -595,7 +595,7 @@
                   h.disconnect();
               }
           }
  -        
  +
           // don't try to get links of a url which is excluded from crawling
           // try to get links of a url which is included for crawling
           if (!isExcludedURL(url.toString()) && isIncludedURL( url.toString() ))
{
  @@ -604,13 +604,13 @@
               + ((url.toExternalForm().indexOf("?") == -1) ? "?" : "&")
               + linkViewQuery;
           }
  -        
  +
           super.contentHandler.startElement(URI, LINK_NODE_NAME, URI+':'+LINK_NODE_NAME,
attributes);
           super.contentHandler.endElement(URI, LINK_NODE_NAME, URI+':'+LINK_NODE_NAME);
  -        
  +
           return result;
       }
  -    
  +
       /**
        * check if URL is a candidate for indexing
        *
  @@ -626,7 +626,7 @@
               }
               return false;
           }
  -        
  +
           final String s = url.toString();
           Iterator i = excludeCrawlingURL.iterator();
           while (i.hasNext()) {
  @@ -643,8 +643,8 @@
           }
           return false;
       }
  -    
  -    
  +
  +
       /**
        * check if URL is a candidate for indexing
        *
  @@ -660,7 +660,7 @@
               }
               return true;
           }
  -        
  +
           final String s = url.toString();
           Iterator i = includeCrawlingURL.iterator();
           while (i.hasNext()) {
  @@ -677,10 +677,10 @@
           }
           return false;
       }
  -    
  +
       public void recycle() {
           super.recycle();
  -        
  +
           this.attributes = null;
           //this.excludeCrawlingURL = null;
       }
  
  
  

----------------------------------------------------------------------
In case of troubles, e-mail:     webmaster@xml.apache.org
To unsubscribe, e-mail:          cocoon-cvs-unsubscribe@xml.apache.org
For additional commands, e-mail: cocoon-cvs-help@xml.apache.org


Mime
View raw message