cocoon-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From upayav...@apache.org
Subject cvs commit: cocoon-2.1/src/java/org/apache/cocoon/bean/helpers DelayedOutputStream.java OutputStreamListener.java
Date Tue, 07 Oct 2003 09:59:17 GMT
upayavira    2003/10/07 02:59:17

  Modified:    src/java/org/apache/cocoon Main.java
               src/java/org/apache/cocoon/bean CocoonBean.java
               src/java/org/apache/cocoon/bean/helpers
                        DelayedOutputStream.java OutputStreamListener.java
  Log:
  Checksum testing of pages once generated. They're now only saved if the checksum is different.
  
  This allows improved uploading, e.g. with Ant, but does not speed the building of the site
itself.
  
  It is accessed by adding a <checksums-uri> node into cli.xconf containing a filename
referring to a file to store the checksums.
  
  Note: this should not be considered stable - I will probably improve upon it which will
change the way it is configured.
  
  Revision  Changes    Path
  1.20      +6 -3      cocoon-2.1/src/java/org/apache/cocoon/Main.java
  
  Index: Main.java
  ===================================================================
  RCS file: /home/cvs/cocoon-2.1/src/java/org/apache/cocoon/Main.java,v
  retrieving revision 1.19
  retrieving revision 1.20
  diff -u -r1.19 -r1.20
  --- Main.java	26 Sep 2003 17:08:08 -0000	1.19
  +++ Main.java	7 Oct 2003 09:59:17 -0000	1.20
  @@ -144,7 +144,8 @@
       private static final String NODE_WORK_DIR = "work-dir";
       private static final String NODE_CONFIG_FILE = "config-file";
       private static final String NODE_URI_FILE = "uri-file";
  -
  +    private static final String NODE_CHECKSUMS_URI = "checksums-uri";
  + 
       private static final String NODE_BROKEN_LINKS = "broken-links";
       private static final String ATTR_BROKEN_LINK_REPORT_TYPE = "type";
       private static final String ATTR_BROKEN_LINK_REPORT_FILE = "file";
  @@ -471,7 +472,6 @@
               if (Main.hasAttribute(root, ATTR_CONFIRM_EXTENSIONS)) {
                   cocoon.setConfirmExtensions(Main.getBooleanAttributeValue(root, ATTR_CONFIRM_EXTENSIONS));
               }
  -
               if (destDir == null || destDir.length() == 0) {
                   destDir = getNodeValue(root, NODE_DEST_DIR);
               }
  @@ -502,6 +502,9 @@
                       } else if (nodeName.equals(NODE_WORK_DIR)) {
                           cocoon.setWorkDir(getNodeValue(node));
   
  +                    } else if (nodeName.equals(NODE_CHECKSUMS_URI)) {
  +                        cocoon.setChecksumURI(getNodeValue(node));
  +                        
                       } else if (nodeName.equals(NODE_AGENT)) {
                           cocoon.setAgentOptions(getNodeValue(node));
   
  
  
  
  1.34      +110 -18   cocoon-2.1/src/java/org/apache/cocoon/bean/CocoonBean.java
  
  Index: CocoonBean.java
  ===================================================================
  RCS file: /home/cvs/cocoon-2.1/src/java/org/apache/cocoon/bean/CocoonBean.java,v
  retrieving revision 1.33
  retrieving revision 1.34
  diff -u -r1.33 -r1.34
  --- CocoonBean.java	6 Oct 2003 14:41:55 -0000	1.33
  +++ CocoonBean.java	7 Oct 2003 09:59:17 -0000	1.34
  @@ -63,14 +63,24 @@
   import org.apache.excalibur.source.ModifiableSource;
   import org.apache.excalibur.source.SourceResolver;
   import org.apache.excalibur.source.Source;
  +import org.apache.excalibur.source.SourceNotFoundException;
  +import org.apache.excalibur.source.SourceUtil;
   
  +import java.io.BufferedReader;
  +import java.io.InputStreamReader;
   import java.io.IOException;
   import java.io.OutputStream;
  +import java.io.OutputStreamWriter;
   import java.io.PrintStream;
  -import java.util.HashMap;
  +import java.io.PrintWriter;
  +import java.security.MessageDigest;
  +import java.security.NoSuchAlgorithmException;
  +
   import java.util.ArrayList;
  +import java.util.HashMap;
   import java.util.Iterator;
   import java.util.List;
  +import java.util.Map;
   
   /**
    * <p>The Cocoon Bean simplifies usage of the Cocoon object. Allows to create,
  @@ -92,6 +102,7 @@
       private boolean followLinks = true;
       private boolean precompileOnly = false;
       private boolean confirmExtension = true;
  +    private boolean checksumCompare = false;
       private String defaultFilename = Constants.INDEX_URI;
       private boolean brokenLinkGenerate = false;
       private String brokenLinkExtension = "";
  @@ -104,7 +115,10 @@
       private List listeners = new ArrayList();
       private boolean verbose;
       SourceResolver sourceResolver;
  -    private Crawler crawler;
  +
  +    private Crawler crawler;    
  +    private String checksumsURI = null;
  +    private Map checksums;
   
       public CocoonBean() {
           this.crawler = new Crawler();
  @@ -169,6 +183,10 @@
           this.brokenLinkExtension = brokenLinkExtension;
       }
   
  +    public void setChecksumURI(String uri) {
  +        this.checksumsURI = uri;
  +    }
  +    
       public boolean followLinks() {
           return followLinks;
       }
  @@ -351,6 +369,8 @@
               this.initialize();
           }
   
  +        readChecksumFile();
  +        
           if (crawler.getRemainingCount()==0) {
               super.precompile();
           } else {
  @@ -364,6 +384,9 @@
                   }
               }
           }
  +        
  +        writeChecksumFile();
  +        
           if (log.isInfoEnabled()) {
                 log.info(
                     "  Memory used: "
  @@ -537,20 +560,24 @@
                       ModifiableSource source = getSource(target);
                       try {
                           pageSize = output.size();
  -                        OutputStream stream = source.getOutputStream();
  -
  -                        output.setFileOutputStream(stream);
  -                        output.flush();
  -                        output.close();
  -                        pageGenerated(target.getSourceURI(),
  -                                      target.getAuthlessDestURI(),
  -                                      pageSize,
  -                                      linkCount,
  -                                      newLinkCount,
  -                                      crawler.getRemainingCount(),
  -                                      crawler.getProcessedCount(),
  -                                      System.currentTimeMillis()- startTimeMillis);
  -
  +                        
  +                        if (this.checksumsURI == null || !isSameContent(output, target))
{
  +                            OutputStream stream = source.getOutputStream();
  +                            output.setFileOutputStream(stream);
  +                            output.flush();
  +                            output.close();
  +                            pageGenerated(target.getSourceURI(), 
  +                                          target.getAuthlessDestURI(), 
  +                                          pageSize,
  +                                          linkCount,
  +                                          newLinkCount,
  +                                          crawler.getRemainingCount(),
  +                                          crawler.getProcessedCount(),
  +                                          System.currentTimeMillis()- startTimeMillis);
  +                        } else {
  +                            output.close();
  +                            pageSkipped(target.getSourceURI(), "Page not changed");
  +                        }
                       } catch (IOException ioex) {
                           log.warn(ioex.toString());
                       } finally {
  @@ -560,7 +587,8 @@
               }
           } catch (Exception rnfe) {
               log.warn("Could not process URI: " + target.getSourceURI());
  -            this.sendBrokenLinkWarning(target.getSourceURI(), "URI not found");
  +            rnfe.printStackTrace();
  +            this.sendBrokenLinkWarning(target.getSourceURI(), "URI not found: "+rnfe.getMessage());
           }
       }
   
  @@ -659,6 +687,70 @@
               return true;
           } else {
               return includeLinkExtensions.contains(target.getExtension());
  +        }
  +    }
  +
  +    /* NB. This is a temporary solution - it may well be replaced by storing the checksum
info
  +     *     in the XML 'report' file, along with details of what pages were created, etc.

  +     */ 
  +    private void readChecksumFile() throws Exception {
  +        checksums = new HashMap();
  +        
  +        try {
  +            Source checksumSource = sourceResolver.resolveURI(checksumsURI);
  +            BufferedReader reader = new BufferedReader(new InputStreamReader(checksumSource.getInputStream()));
  +            String line;
  +            int lineNo=0;
  +            while ((line = reader.readLine())!=null) {
  +                lineNo++;
  +                if (line.trim().startsWith("#") || line.trim().length()==0 ) {
  +                    continue;
  +                }
  +                if (line.indexOf("\t")==-1) { 
  +                    throw new ProcessingException("Missing tab at line "+lineNo+" of "
+ checksumsURI);
  +                }
  +                String filename = line.substring(0,line.indexOf("\t"));
  +                String checksum = line.substring(line.indexOf("\t")+1);
  +                checksums.put(filename, checksum);
  +            }
  +            reader.close();
  +        } catch (SourceNotFoundException e) {
  +            // return leaving checksums map empty
  +        }
  +    }
  +    
  +    private void writeChecksumFile() throws Exception {
  +        Source checksumSource = sourceResolver.resolveURI(checksumsURI);
  +        if (!(checksumSource instanceof ModifiableSource)) {
  +            throw new ProcessingException("Checksum file is not Modifiable:" + checksumSource);
  +        }
  +        ModifiableSource source = (ModifiableSource) checksumSource;
  +        PrintWriter writer = new PrintWriter(new OutputStreamWriter(source.getOutputStream()));
  +        Iterator i = checksums.keySet().iterator();
  +        while (i.hasNext()){
  +            String key = (String) i.next();
  +            String checksum = (String) checksums.get(key);
  +            writer.println(key + "\t" + checksum);
  +        }
  +        writer.close();
  +    }
  +
  +    private boolean isSameContent(DelayedOutputStream stream, Target target) {
  +        try {
  +            MessageDigest md5 = MessageDigest.getInstance("MD5");
  +            md5.update(stream.getContent());
  +            String streamDigest = SourceUtil.encodeBASE64(new String(md5.digest()));
  +            String targetDigest = (String)checksums.get(target.getSourceURI());
  +            
  +            if (streamDigest.equals(targetDigest)) {
  +                return true;
  +            } else {
  +                checksums.put(target.getSourceURI(), streamDigest);
  +                return false;
  +            }
  +        } catch (NoSuchAlgorithmException e) {
  +            // or do something:
  +            return false;
           }
       }
   }
  
  
  
  1.5       +13 -1     cocoon-2.1/src/java/org/apache/cocoon/bean/helpers/DelayedOutputStream.java
  
  Index: DelayedOutputStream.java
  ===================================================================
  RCS file: /home/cvs/cocoon-2.1/src/java/org/apache/cocoon/bean/helpers/DelayedOutputStream.java,v
  retrieving revision 1.4
  retrieving revision 1.5
  diff -u -r1.4 -r1.5
  --- DelayedOutputStream.java	18 Sep 2003 12:11:50 -0000	1.4
  +++ DelayedOutputStream.java	7 Oct 2003 09:59:17 -0000	1.5
  @@ -250,6 +250,7 @@
               throw new IOException("No outputstream available!");
           }
       }
  +
       /**
        * Gets the size of the content of the current output stream
        */
  @@ -258,5 +259,16 @@
               return baos.size();
           }
           return 0;
  +    }
  +
  +    /**
  +     * Return the contents of the stream as a byte array
  +     */
  +    public byte[] getContent() {
  +        if (baos != null) {
  +            return baos.toByteArray();
  +        } else {
  +            return null;
  +        }
       }
   }
  
  
  
  1.6       +35 -33    cocoon-2.1/src/java/org/apache/cocoon/bean/helpers/OutputStreamListener.java
  
  Index: OutputStreamListener.java
  ===================================================================
  RCS file: /home/cvs/cocoon-2.1/src/java/org/apache/cocoon/bean/helpers/OutputStreamListener.java,v
  retrieving revision 1.5
  retrieving revision 1.6
  diff -u -r1.5 -r1.6
  --- OutputStreamListener.java	6 Oct 2003 12:40:14 -0000	1.5
  +++ OutputStreamListener.java	7 Oct 2003 09:59:17 -0000	1.6
  @@ -34,7 +34,7 @@
    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    FITNESS  FOR A PARTICULAR  PURPOSE ARE  DISCLAIMED.  IN NO  EVENT SHALL  THE
  - APACHE SOFTWARE  FOUNDATION  OR ITS CONTRIBUTORS  BE LIABLE FOR  ANY DIRECT, 
  + APACHE SOFTWARE  FOUNDATION  OR ITS CONTRIBUTORS  BE LIABLE FOR  ANY DIRECT,
    INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL  DAMAGES (INCLU-
    DING, BUT NOT LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS
    OF USE, DATA, OR  PROFITS; OR BUSINESS  INTERRUPTION)  HOWEVER CAUSED AND ON
  @@ -50,14 +50,15 @@
   */
   package org.apache.cocoon.bean.helpers;
   
  -import java.util.ArrayList;
  -import java.util.Iterator;
  -import java.util.List;
   import java.io.File;
   import java.io.FileWriter;
   import java.io.IOException;
   import java.io.OutputStream;
   import java.io.PrintWriter;
  +import java.text.DecimalFormat;
  +import java.util.ArrayList;
  +import java.util.Iterator;
  +import java.util.List;
   
   import org.apache.cocoon.bean.BeanListener;
   
  @@ -69,58 +70,58 @@
    * @version CVS $Id$
    */
   public class OutputStreamListener implements BeanListener {
  -    
  +
       private final PrintWriter writer;
       private final List brokenLinks = new ArrayList();
  -    private final long startTimeMillis;    
  +    private final long startTimeMillis;
       private String reportFile = null;
       private String reportType = "text";
       private long siteSize = 0L;
       private int sitePages = 0;
  -   
  +
       public OutputStreamListener(OutputStream os) {
           writer = new PrintWriter(os);
           startTimeMillis = System.currentTimeMillis();
       }
  -    
  +
       public void setReportFile(String filename) {
           reportFile = filename;
       }
   
       public void setReportType(String type) {
  -        reportType = type;     
  +        reportType = type;
       }
  -    
  +
       public void pageGenerated(String sourceURI,
  -                              String destinationURI, 
  +                              String destinationURI,
                                 int pageSize,
  -                              int linksInPage, 
  -                              int newLinksInPage, 
  -                              int pagesRemaining, 
  -                              int pagesComplete, 
  +                              int linksInPage,
  +                              int newLinksInPage,
  +                              int pagesRemaining,
  +                              int pagesComplete,
                                 long timeTaken) {
           this.siteSize += pageSize;
           this.sitePages++;
  -        
  +
           double time = (((double)timeTaken)/1000);
  -        
  +
           String size;
           if (pageSize < 1024) {
               size = pageSize + "b";
           } else {
               size = ((float)((int)(pageSize/102.4)))/10 + "Kb";
           }
  -        
  +
           if (linksInPage == -1) {
               this.print("* " + sourceURI);
           } else {
  -            this.print(pad(12, "* [" + pagesComplete + "/" + pagesRemaining + "] ") + 
  +            this.print(pad(12, "* [" + pagesComplete + "/" + pagesRemaining + "] ") +
                          pad(10, "[" + newLinksInPage + "/" + linksInPage + "] ") +
                          pad(7,time + "s ") +
                          pad(7, size) + " " +
                          sourceURI);
  -        }     
  -           
  +        }
  +
       }
       public void messageGenerated(String msg) {
           this.print(msg);
  @@ -133,7 +134,7 @@
       public void brokenLinkFound(String uri, String parentURI, String message, Throwable
t) {
           this.print(pad(42,"X [0] ")+uri+"\tBROKEN: "+message);
           brokenLinks.add(uri + "\t" + message);
  -        
  +
   //            StringWriter sw = new StringWriter();
   //            t.printStackTrace(new PrintWriter(sw));
   //            System.out.println(sw.toString());
  @@ -143,16 +144,17 @@
       public void pageSkipped(String uri, String message) {
           this.print(pad(37, "^ ") + uri);
       }
  -    
  +
       public void complete() {
           outputBrokenLinks();
   
           long duration = System.currentTimeMillis() - startTimeMillis;
  -        
  -        this.print("Total time: " + 
  -                   (duration / 60000) + " minutes " + 
  -                   (duration % 60000)/1000 + " seconds, " + 
  -                   " Site size: " + this.siteSize +
  +        DecimalFormat df = new DecimalFormat("###,###,##0");
  +
  +        this.print("Total time: " +
  +                   (duration / 60000) + " minutes " +
  +                   (duration % 60000)/1000 + " seconds, " +
  +                   " Site size: " + df.format(this.siteSize) +
                      " Site pages: " + this.sitePages);
           this.close();
       }
  @@ -160,7 +162,7 @@
       public boolean isSuccessful() {
           return brokenLinks.size() == 0;
       }
  -    
  +
       private void outputBrokenLinks() {
           if (reportFile == null) {
               return;
  @@ -170,7 +172,7 @@
               outputBrokenLinksAsXML();
           }
       }
  -    
  +
       private void outputBrokenLinksAsText() {
           PrintWriter writer;
           try {
  @@ -219,12 +221,12 @@
           }
           return str;
       }
  -    
  +
       private void print(String message) {
           writer.println(message);
           writer.flush();
       }
  -    
  +
       private void close() {
           writer.close();
       }
  
  
  

Mime
View raw message