incubator-droids-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ol...@apache.org
Subject svn commit: r711718 - in /incubator/droids/trunk/src: java/org/apache/droids/api/ java/org/apache/droids/handle/ java/org/apache/droids/helper/factories/ java/org/apache/droids/impl/ java/org/apache/droids/net/ java/org/apache/droids/robot/crawler/ jav...
Date Wed, 05 Nov 2008 22:12:42 GMT
Author: olegk
Date: Wed Nov  5 14:12:41 2008
New Revision: 711718

URL: http://svn.apache.org/viewvc?rev=711718&view=rev
Log:
DROIDS-28: improved exception handling

Added:
    incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java
    incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java
    incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
Modified:
    incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java
    incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java
    incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java
    incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java
    incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java
    incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java
    incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
    incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
    incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
    incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
    incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java
    incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
    incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
    incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
    incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
    incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java
    incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java

Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Droid.java Wed Nov  5 14:12:41 2008
@@ -40,7 +40,7 @@
    * 
    * @throws DroidsException
    */
-  void init();
+  void init() throws DroidsException;
 
   /**
    * Invoke an instance of the worker used in the droid

Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Handler.java Wed Nov  5 14:12:41
2008
@@ -16,9 +16,12 @@
  */
 package org.apache.droids.api;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 
+import org.apache.droids.exception.DroidsException;
+
 /**
  * A handler is a component that uses the stream, the parse and url to invoke
  * arbitrary business logic on the objects.
@@ -36,5 +39,6 @@
    *                the parse object from a former processing step
    * @throws Exception
    */
-  void handle(InputStream openStream, URL url, Parse parse) throws Exception;
+  void handle(InputStream openStream, URL url, Parse parse) 
+    throws IOException, DroidsException;
 }

Added: incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java (added)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionHandler.java Wed Nov
 5 14:12:41 2008
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+public interface TaskExceptionHandler
+{
+
+  TaskExceptionResult handleException(Exception ex);
+  
+}

Added: incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java (added)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/TaskExceptionResult.java Wed Nov
 5 14:12:41 2008
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+public enum TaskExceptionResult
+{
+  
+  IGNORE, WARN, FATAL
+  
+}

Modified: incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/api/Worker.java Wed Nov  5 14:12:41
2008
@@ -16,6 +16,10 @@
  */
 package org.apache.droids.api;
 
+import java.io.IOException;
+
+import org.apache.droids.exception.DroidsException;
+
 
 /**
  * A worker is the unit that is doing the actual work. A {@link Droid} is the
@@ -27,6 +31,6 @@
  */
 public interface Worker<T extends Task> {
   
-  void execute( final T task );
+  void execute( final T task ) throws DroidsException, IOException;
   
 }

Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/Save.java Wed Nov  5 14:12:41
2008
@@ -52,7 +52,7 @@
    *      java.net.URL, org.apache.droids.api.Parse)
    */
   public void handle(InputStream stream, URL urlToHandle, Parse parse)
-      throws Exception {
+      throws IOException {
     this.url = urlToHandle;
     writeOutput(stream);
   }

Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/Sysout.java Wed Nov  5 14:12:41
2008
@@ -48,7 +48,7 @@
    * @see org.apache.droids.api.Handler#handle(java.io.InputStream,
    *      java.net.URL, org.apache.droids.api.Parse)
    */
-  public void handle(InputStream stream, URL url, Parse parse) throws Exception {
+  public void handle(InputStream stream, URL url, Parse parse) throws IOException {
     writeOutput(stream);
   }
 

Modified: incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/handle/WriterHandler.java Wed Nov  5
14:12:41 2008
@@ -20,8 +20,7 @@
 import java.io.Reader;
 import java.io.Writer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.droids.helper.Loggable;
 
 /**
  * Wrapper that allows you to pipe a stream from a reader to a writer via a
@@ -30,11 +29,9 @@
  * @version 1.0
  * 
  */
-public class WriterHandler {
+public class WriterHandler extends Loggable {
 
   private static final int BUFFER_SIZE = 1024;
-  protected final Log log = LogFactory.getLog(this.getClass()
-      .getCanonicalName());
 
   /**
    * Pipes everything from the reader to the writer via a buffer

Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
(original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/GenericFactory.java
Wed Nov  5 14:12:41 2008
@@ -18,8 +18,7 @@
 
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.droids.helper.Loggable;
 
 /**
  * Basically all factories till now extend this generic factory. The core is a
@@ -28,9 +27,7 @@
  * @version 1.0
  * 
  */
-public class GenericFactory<T> {
-
-  protected final Log log = LogFactory.getLog(this.getClass().getCanonicalName());
+public class GenericFactory<T> extends Loggable {
 
   private Map<String,T> map = null;
 

Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
(original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/HandlerFactory.java
Wed Nov  5 14:12:41 2008
@@ -25,6 +25,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.droids.api.Handler;
 import org.apache.droids.api.Parse;
+import org.apache.droids.exception.DroidsException;
 
 /**
  * Factory that will traverse all registered handler and execute them.
@@ -46,28 +47,21 @@
    *                the underlying parse object
    * @return false if we found a problem, true if all went well
    */
-  public boolean handle(InputStream stream, URL url, Parse parse) {
+  public boolean handle(InputStream stream, URL url, Parse parse) 
+      throws DroidsException, IOException {
     byte[] streamCopy = null;
     if(stream==null){
       return false;
     }
-    try {
-      ByteArrayOutputStream out = new ByteArrayOutputStream();
-      IOUtils.copy(stream, out);
-      streamCopy = out.toByteArray();
-    } catch (IOException e) {
-      log.fatal("Handler setup has thrown an error.", e);
-    }
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copy(stream, out);
+    streamCopy = out.toByteArray();
     
     for (Handler handler : getMap().values()) {
       if (streamCopy == null) {
         return false;
       }
-      try {
-        handler.handle(new ByteArrayInputStream(streamCopy), url, parse);
-      } catch (Exception e) {
-        log.fatal("Handler \"" + handler + "\" has thrown an error.", e);
-      }
+      handler.handle(new ByteArrayInputStream(streamCopy), url, parse);
     }
     return true;
   }

Modified: incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
(original)
+++ incubator/droids/trunk/src/java/org/apache/droids/helper/factories/ProtocolFactory.java
Wed Nov  5 14:12:41 2008
@@ -38,7 +38,8 @@
    * @return ready to use protocol plugin or null if non have been found
    * @throws ProtocolNotFoundException
    */
-  public Protocol getProtocol(String uri) throws ProtocolNotFoundException {
+  public Protocol getProtocol(String uri) 
+      throws MalformedURLException, ProtocolNotFoundException {
     URL url = null;
     Protocol protocol = null;
     try {
@@ -48,8 +49,6 @@
         throw new ProtocolNotFoundException(uri);
       }
       protocol = getMap().get(protocolName);
-    } catch (MalformedURLException e) {
-      throw new ProtocolNotFoundException(uri, e.toString());
     } catch (ProtocolNotFoundException e) {
       throw new ProtocolNotFoundException(uri, e.toString());
     }

Added: incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java?rev=711718&view=auto
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
(added)
+++ incubator/droids/trunk/src/java/org/apache/droids/impl/DefaultTaskExceptionHandler.java
Wed Nov  5 14:12:41 2008
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.impl;
+
+import java.io.IOException;
+
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
+import org.apache.droids.exception.DroidsException;
+
+public class DefaultTaskExceptionHandler implements TaskExceptionHandler {
+
+  public TaskExceptionResult handleException(Exception ex) {
+    if (ex instanceof IOException) {
+      return TaskExceptionResult.WARN;
+    } else if (ex instanceof DroidsException) {
+      return TaskExceptionResult.WARN;
+    } else {
+      return TaskExceptionResult.FATAL;
+    }
+  }
+  
+}

Modified: incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/impl/MultiThreadedTaskMaster.java Wed
Nov  5 14:12:41 2008
@@ -25,6 +25,8 @@
 import org.apache.droids.api.DelayTimer;
 import org.apache.droids.api.Droid;
 import org.apache.droids.api.Task;
+import org.apache.droids.api.TaskExceptionHandler;
+import org.apache.droids.api.TaskExceptionResult;
 import org.apache.droids.api.TaskMaster;
 import org.apache.droids.api.TaskQueue;
 import org.apache.droids.api.Worker;
@@ -46,6 +48,8 @@
   private Date finishedWorking = null;
   private int completedTask = 0;
   private T lastCompletedTask = null;
+
+  private TaskExceptionHandler exHandler;
   
   /**
    * The queue has been initialized
@@ -80,7 +84,27 @@
         if (log.isInfoEnabled()) {
           log.info("Worker \"" + id + "\" has finished.");
         }
-        if (runningWorker.size() == 0 & !queue.hasNext()) {
+
+        boolean terminate = false;
+        
+        Exception ex = worker.getException();
+        if (ex != null) {
+          TaskExceptionResult result = TaskExceptionResult.WARN;  
+          if (exHandler != null) {
+            result = exHandler.handleException(ex); 
+          }
+          switch (result) {
+          case WARN:
+            log.warn(ex.getMessage());
+            break;
+          case FATAL:
+            log.warn(ex.getMessage());
+            terminate = true;
+            break;
+          }
+        }
+        
+        if (terminate || (runningWorker.size() == 0 & !queue.hasNext())) {
           shutdownAndAwaitTermination();
           
           long elapsed = System.currentTimeMillis() - startedWorking.getTime();
@@ -115,22 +139,21 @@
       if( activeWorkers >= getMaxThreads() ) {
         return;  // don't make a new runner...
       }
-      
-      try {
-        WorkerRunner runner = new WorkerRunner();
-        runningWorker.put(runner.getId(), runner);
-        pool.execute(runner);
-        if (log.isInfoEnabled()) {
-          log.info("starting " + runner.getId());
-        }
-      } 
-      catch (Exception ex) {
-        log.error(ex.getMessage(), ex);
-      } 
+      WorkerRunner runner = new WorkerRunner();
+      runningWorker.put(runner.getId(), runner);
+      pool.execute(runner);
+      if (log.isInfoEnabled()) {
+        log.info("starting " + runner.getId());
+      }
     }
   }
 
 
+  public final void setExceptionHandler(TaskExceptionHandler exHandler) {
+    this.exHandler = exHandler;
+  }
+  
+
   public DelayTimer getDelayTimer() {
     return delayTimer;
   }
@@ -189,6 +212,7 @@
   class WorkerRunner extends Thread {
     
     T task;
+    Exception exception;
     
     @Override
     public void run() {
@@ -206,13 +230,22 @@
           }
 
           Worker<T> worker = droid.getNewWorker();
-          worker.execute( task );
+          try {
+            worker.execute( task );
+          } catch (Exception ex) {
+            exception = ex;
+          }
         }
       }
       finally {
         finishedWorker( getId() );
       }
     }
+    
+    public Exception getException() {
+      return exception;
+    }
+    
   }
 
   public int getCompletedTasks() {

Modified: incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/net/RegexURLFilter.java Wed Nov  5 14:12:41
2008
@@ -58,31 +58,17 @@
   /**
    * @param file
    */
-  public void setFile(String file) {
+  public void setFile(String file) throws IOException {
     URL url = null;
-    try {
-      if (file.startsWith("classpath:/")) {
-        url = this.getClass().getResource(
-            file.substring("classpath:/".length() - 1));
-      } else {
-        url = (file.contains(":/") ? new URL(file) : new URL("file://" + file));
-      }
-      log.debug("url " + url);
-      Reader reader = null;
-      try {
-        reader = new InputStreamReader(url.openStream());
-      } catch (RuntimeException e) {
-        log.fatal("Can't create reader for url: " + url);
-      }
-      if (reader != null) {
-        rules = readRulesFile(reader);
-      } else {
-        log.fatal("Can't find resource: " + file);
-      }
-    } catch (Exception e1) {
-      log.fatal("File  " + file + " has thrown an exception:\n" + e1);
+    if (file.startsWith("classpath:/")) {
+      url = this.getClass().getResource(
+          file.substring("classpath:/".length() - 1));
+    } else {
+      url = (file.contains(":/") ? new URL(file) : new URL("file://" + file));
     }
-
+    log.debug("url " + url);
+    Reader reader = new InputStreamReader(url.openStream());
+    rules = readRulesFile(reader);
   }
 
   private RegexRule[] readRulesFile(Reader reader) throws IOException {

Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingDroid.java Wed
Nov  5 14:12:41 2008
@@ -17,12 +17,15 @@
 package org.apache.droids.robot.crawler;
 
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.droids.AbstractDroid;
 import org.apache.droids.LinkTask;
 import org.apache.droids.api.*;
+import org.apache.droids.exception.InvalidTaskException;
 import org.apache.droids.helper.factories.HandlerFactory;
 import org.apache.droids.helper.factories.ParserFactory;
 import org.apache.droids.helper.factories.ProtocolFactory;
@@ -48,17 +51,18 @@
     this.initialLocations = initialLocations;
   }
   
-  public void init() {
+  public void init() throws InvalidTaskException {
     if( initialLocations == null || initialLocations.isEmpty() ) {
-      throw new RuntimeException( "WebCrawlerDroid requires at least one starting file" );
+      throw new IllegalStateException( "WebCrawlerDroid requires at least one starting file"
);
     }
-    for( String uri : initialLocations ) {
+    for( String location : initialLocations ) {
+      URI uri;
       try {
-        queue.merge( new LinkTask( null, new URI(uri), 0 ) );
-      }
-      catch( Exception ex ) {
-        throw new RuntimeException( ex );
+        uri = new URI(location);
+      } catch (URISyntaxException ex) {
+        throw new InvalidTaskException("Invalid lication: " + location);
       }
+      queue.merge( new LinkTask( null, uri, 0 ) );
     }
   }
   
@@ -111,25 +115,21 @@
   //------------------------------------------------------------------
   //------------------------------------------------------------------
   
-  public static void main( String[] args )
+  public static void main( String[] args ) throws Exception
   {
-    try {
-      MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
-      taskMaster.setMaxThreads( 3 );
-      
-      TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-      
-      Collection<String> locations = new ArrayList<String>();
-      locations.add( args[0] );
-
-      CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
-      simple.setInitialLocations( locations );
-      simple.init();
-      simple.start();  // TODO? perhaps start internally calls init()?
-    } 
-    catch (Exception e) {
-      e.printStackTrace();
-    }
+    MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
+    taskMaster.setMaxThreads( 3 );
+    
+    TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
+    
+    Collection<String> locations = new ArrayList<String>();
+    locations.add( args[0] );
+
+    CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
+    simple.setInitialLocations( locations );
+    simple.init();
+    simple.start();  // TODO? perhaps start internally calls init()?
+    simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
   }
 }
 

Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/crawler/CrawlingWorker.java Wed
Nov  5 14:12:41 2008
@@ -18,7 +18,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Collection;
 import java.util.HashMap;
@@ -29,6 +28,7 @@
 import org.apache.droids.api.Parser;
 import org.apache.droids.api.Protocol;
 import org.apache.droids.api.Worker;
+import org.apache.droids.exception.DroidsException;
 import org.apache.droids.helper.Loggable;
 import org.apache.droids.helper.factories.URLFiltersFactory;
 
@@ -41,7 +41,7 @@
     this.droid = droid;
   }
 
-  public void execute(Link link) 
+  public void execute(Link link) throws DroidsException, IOException
   {
     InputStream openStream = null;
     final String userAgent = this.getClass().getCanonicalName();
@@ -82,29 +82,22 @@
             + " bots are not allowed for this url.");
       }
     } 
-    catch (Exception e) {
-      e.printStackTrace();
-    }
     finally{
       try {
         if (openStream != null) {
           openStream.close();
         }
-      } catch (IOException e) {
-        e.printStackTrace();
+      } catch (IOException ex) {
+        log.error("Error closing stream", ex);
       }
     }
   }
   
-  protected void handle( Parse parse, InputStream openStream, Link link )
+  protected void handle( Parse parse, InputStream openStream, Link link ) 
+      throws DroidsException, IOException
   {
     String url = link.getId();
-    try {
-      droid.getHandlerFactory().handle(openStream, new URL(url), parse);
-    } 
-    catch (MalformedURLException e) {
-      e.printStackTrace();
-    }
+    droid.getHandlerFactory().handle(openStream, new URL(url), parse);
   }
   
   protected Collection<Link> getFilteredOutlinks( Parse parse )

Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java (original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/FileWorker.java Wed Nov
 5 14:12:41 2008
@@ -32,18 +32,12 @@
     this.queue = queue;
   }
 
-  public void execute(FileTask task) 
+  public void execute(FileTask task) throws InvalidTaskException
   {
     File file = task.getFile();
     if( file.isDirectory() ) {
-      // Add every file in the directory to the Queue
-      try {
-        for( File f : file.listFiles() ) {
-          queue.merge( new FileTask( f, task.getDepth()+1 ) );
-        }
-      } 
-      catch (InvalidTaskException e) {
-        e.printStackTrace();
+      for( File f : file.listFiles() ) {
+        queue.merge( new FileTask( f, task.getDepth()+1 ) );
       }
     }
     else {

Modified: incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
(original)
+++ incubator/droids/trunk/src/java/org/apache/droids/robot/walker/SimpleWalkingDroid.java
Wed Nov  5 14:12:41 2008
@@ -19,6 +19,7 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.droids.AbstractDroid;
 import org.apache.droids.exception.InvalidTaskException;
@@ -39,23 +40,18 @@
     this.initialFiles = initialFiles;
   }
   
-  public void init() {
+  public void init() throws InvalidTaskException {
     if( initialFiles == null || initialFiles.isEmpty() ) {
-      throw new RuntimeException( "FileSystemWalker requires at least one starting file"
);
+      throw new IllegalStateException( "FileSystemWalker requires at least one starting file"
);
     }
-    try {
-      for( File file : initialFiles ) {
-        queue.merge( new FileTask( file, 0 ) );
-      }
-    } 
-    catch (InvalidTaskException e) {
-      e.printStackTrace();
+    for( File file : initialFiles ) {
+      queue.merge( new FileTask( file, 0 ) );
     }
   }
   
   public void finished()
   {
-    System.out.println( "FINISHED!!!" );
+    log.info( "FINISHED!!!" );
   }
 
   public FileWorker getNewWorker() {
@@ -65,7 +61,7 @@
   //------------------------------------------------------------------
   //------------------------------------------------------------------
   
-  public static void main( String[] args )
+  public static void main( String[] args ) throws Exception 
   {
     MultiThreadedTaskMaster<FileTask> taskMaster = new MultiThreadedTaskMaster<FileTask>();
     taskMaster.setMaxThreads( 3 );
@@ -79,6 +75,7 @@
     simple.setInitialFiles( files );
     simple.init();
     simple.start();  // TODO? perhaps start internally calls init()?
+    simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
   }
 }
 

Modified: incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java (original)
+++ incubator/droids/trunk/src/test/java/org/apache/droids/examples/SimpleRuntime.java Wed
Nov  5 14:12:41 2008
@@ -29,6 +29,7 @@
 import org.apache.droids.helper.factories.ParserFactory;
 import org.apache.droids.helper.factories.ProtocolFactory;
 import org.apache.droids.helper.factories.URLFiltersFactory;
+import org.apache.droids.impl.DefaultTaskExceptionHandler;
 import org.apache.droids.impl.MultiThreadedTaskMaster;
 import org.apache.droids.impl.SimpleTaskQueue;
 import org.apache.droids.net.RegexURLFilter;
@@ -99,6 +100,7 @@
     MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
     taskMaster.setMaxThreads( 5 );
     taskMaster.setDelayTimer( simpleDelayTimer );
+    taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
     
     CrawlingDroid helloCrawler = new CrawlingDroid( simpleQueue, taskMaster );
     helloCrawler.setFiltersFactory(filtersFactory);

Modified: incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=711718&r1=711717&r2=711718&view=diff
==============================================================================
--- incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java (original)
+++ incubator/droids/trunk/src/test/java/org/apache/droids/impl/TestSimpleDroid.java Wed Nov
 5 14:12:41 2008
@@ -100,7 +100,7 @@
     HandlerFactory handlerFactory = new HandlerFactory();
     Handler defaultHandler = new Handler() {
 
-      public void handle(InputStream openStream, URL url, Parse parse) throws Exception {
+      public void handle(InputStream openStream, URL url, Parse parse) {
         visitedLinks.add(url); 
       }
       



Mime
View raw message