Mailing-List: contact commits-help@labs.apache.org; run by ezmlm
Precedence: bulk
Reply-To: labs@labs.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r679332 - in /labs/droids/trunk/src:
 core/java/org/apache/droids/ core/java/org/apache/droids/helper/factories/
 core/java/org/apache/droids/parse/html/ core/java/org/apache/droids/queue/
 examples/java/org/apache/droids/examples/
Date: Thu, 24 Jul 2008 09:43:59 -0000
To: commits@labs.apache.org
From: thorsten@apache.org
Message-Id: <20080724094400.224D12388987@eris.apache.org>

Author: thorsten
Date: Thu Jul 24 02:43:57 2008
New Revision: 679332

URL: http://svn.apache.org/viewvc?rev=679332&view=rev
Log:
Refactoring code

Added:
    labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java   (with props)
Modified:
    labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
    labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
    labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml
    labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/HandlerFactory.java
    labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/URLFiltersFactory.java
    labs/droids/trunk/src/core/java/org/apache/droids/parse/html/HtmlParser.java
    labs/droids/trunk/src/core/java/org/apache/droids/queue/QueueBean.java
    labs/droids/trunk/src/core/java/org/apache/droids/queue/Simple.java
    labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
    labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java

Added: labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java?rev=679332&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java Thu Jul 24 02:43:57 2008
@@ -0,0 +1,245 @@
+package org.apache.droids;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.droids.api.Droid;
+import org.apache.droids.api.Queue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.exception.DroidsException;
+
+/**
+ * @author thorsten
+ * @version 1.0
+ *
+ */
+public abstract class AbstractDroid implements Droid{
+  /* 
+   * @see org.apache.droids.api.Droid#finishedWorker(long)
+   */
+  public void finishedWorker(long id)
+      throws DroidsException {
+    synchronized (this) {
+      Worker worker = getRunningWorker().get(id);
+      if (null != worker) {
+        //int y = worker.getDepth() + 1;
+        getPool().remove(worker);
+        getRunningWorker().remove(id);
+        Core.threadMessage("Worker \"" + id + "\" has finished.");
+        if (getRunningWorker().size() == 0 & !getQueue().hasNext()) {
+          shutdownAndAwaitTermination();
+          Core.threadMessage("All threads has finished.");
+        } else if (getQueue().hasNext()) {
+          for (int i = 0; i < (getMaxThreads() - getPool().getActiveCount()); i++) {
+            if (!getQueue().hasNext()) {
+              break;
+            }
+            startWorkers();
+          }
+        }
+      }
+    }
+       
+      }
+
+  /**
+   * Will start a new worker.
+   * @return the id of the worker we just started.
+   */
+  public abstract long startWorkers();
+  
+  /**
+   * Get the default worker for the class.
+   * @return Worker that should do the job.
+   * @throws DroidsException
+   */
+  public abstract Worker getWorker() throws DroidsException;
+
+  private Core core = null;
+  /* 
+   * @see org.apache.droids.api.Droid#getCore()
+   */
+  public Core getCore() {
+    return core;
+  }
+
+  /**
+   * Set the fully configured core and inject it in the
+   * @param core
+   */
+  public void setCore(Core core) {
+    this.core = core;
+  }
+
+  private ThreadPoolExecutor pool = null;
+
+  /**
+   * Get our pool.
+   * @return our pool
+   */
+  public ThreadPoolExecutor getPool() {
+    synchronized (pool) {
+      return pool;
+    }
+  }
+
+  /**
+   * Set our pool.
+   * @param pool the new pool.
+   */
+  public void setPool(ThreadPoolExecutor pool) {
+    synchronized (this) {
+      this.pool = pool;
+    }
+  }
+
+  private ConcurrentHashMap<Long, Worker> runningWorker = null;
+  /**
+   * Return the map of running workers
+   * @return map of running workers
+   */
+  public ConcurrentHashMap<Long, Worker> getRunningWorker() {
+    synchronized (runningWorker) {
+      return runningWorker;
+    }
+  }
+
+  /**
+   * Set the map of running workers
+   * @param runningWorker new map of running workers
+   */
+  public void setRunningWorker(ConcurrentHashMap<Long, Worker> runningWorker) {
+    this.runningWorker = runningWorker;
+  }
+
+  private int runningThreads = 0;
+  /**
+   * To set the number of running threads.
+   * @param runningThreads
+   */
+  public void setRunningThreads(int runningThreads) {
+    this.runningThreads = runningThreads;
+  }
+
+  /**
+   * Get number of currently running threads
+   * @return number of currently running threads
+   */
+  public int getRunningThreads() {
+    return runningThreads;
+  }
+
+  private String taskDate = "";
+
+  /**
+   * When did the task showed up the first time in the queue
+   * 
+   * @return the date when the task registered with the queue
+   */
+  public String getTaskDate() {
+    return taskDate;
+  }
+
+  /**
+   * When did the task showed up the first time in the queue
+   * 
+   * @param taskDate the date when the task registered with the queue
+   */
+  public void setTaskDate(String taskDate) {
+    synchronized (this) {
+      this.taskDate = taskDate;
+    }
+  }
+  
+  private int freeSlots=0;
+
+  /**
+   * Get number of slots that we have currently
+   * open to accept new workers.
+   * @return number of slots that are waiting to serve
+   */
+  public int getFreeSlots() {
+    synchronized (this) {
+      return freeSlots;
+    }
+  }
+
+  /**
+   * Set number of slots that we have currently
+   * open to accept new workers 
+   * @param freeSlots number of slots that are waiting to serve
+   */
+  public void setFreeSlots(int freeSlots) {
+    synchronized (this) {
+      this.freeSlots = freeSlots;
+    }
+  }
+
+  private int maxThreads = 0;
+
+  /**
+   * Adjust number of allowed threads
+   * @param maxThreads
+   */
+  public  void setMaxThreads(int maxThreads) {
+    synchronized (this) {
+      this.maxThreads = maxThreads;
+    }
+  }
+
+  /**
+   * Get number of maximum allowed threads
+   * @return the number of maximum threads that we allow
+   */
+  public int getMaxThreads() {
+    return maxThreads;
+  }
+
+  private Queue queue = null;
+  /* 
+   * @see org.apache.droids.api.Droid#setQueue(Queue)
+   */
+  public void setQueue(Queue queue) {
+    synchronized (this) {
+      this.queue = queue;
+    }
+  }
+
+  /**
+   * Get the queue implementation that we want to use.
+   * @return
+   */
+  protected Queue getQueue() {
+    synchronized (this) {
+      return queue;
+    }
+  }
+
+  /**
+   * Shutdown all threads, close the pools and leave.
+   * If it is not working by asking nice to shutdown just kill all
+   * threads.
+   */
+  protected void shutdownAndAwaitTermination() {
+    Core.threadMessage("SHUTTING DOWN");
+    getPool().shutdown(); // Disable new tasks from being submitted
+    try {
+      // Wait a while for existing tasks to terminate
+      if (!getPool().awaitTermination(1, TimeUnit.SECONDS)) {
+        Core.threadMessage("SHUT DOWN NOW");
+        getPool().shutdownNow(); // Cancel currently executing tasks
+        // Wait a while for tasks to respond to being cancelled
+        if (!getPool().awaitTermination(1, TimeUnit.SECONDS)) {
+          Core.threadMessage("Pool did not terminate");
+        }
+      }
+    } catch (InterruptedException ie) {
+      // (Re-)Cancel if current thread also interrupted
+      getPool().shutdownNow();
+      // Preserve interrupt status
+      Thread.currentThread().interrupt();
+    }
+  }
+
+}
\ No newline at end of file

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java Thu Jul 24 02:43:57 2008
@@ -23,8 +23,6 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.droids.api.Droid;
-import org.apache.droids.api.Queue;
 import org.apache.droids.api.Task;
 import org.apache.droids.api.Worker;
 import org.apache.droids.exception.DroidsException;
@@ -37,17 +35,11 @@
  * 
  * Extend this method for your own Droid. 
  * @author thorsten
- *
+ * @version 1.0
  */
-public class DefaultCrawler implements Droid {
-  protected Queue queue;
-
-  protected String taskDate;
-
-  protected int freeSlots;
-
-  private ConcurrentHashMap<Long, Worker> runningWorker;
-
+public class DefaultCrawler extends AbstractDroid  {
+  private static final int THREAD_SLEEP = 4000;
+  private static final long KEEP_ALIVE = 50000L;
   /**
    * Do the work (whatever it is defined 
    * in the Droid and their workers)
@@ -56,196 +48,86 @@
     setRunningThreads (0);
     setTaskDate( new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System
         .currentTimeMillis())));
-    setFreeSlots (maxThreads - runningThreads);
-    setPool( new ThreadPoolExecutor(freeSlots, freeSlots, 50000L,
+    setFreeSlots (getMaxThreads() - getRunningThreads());
+    setPool( new ThreadPoolExecutor(getFreeSlots(), getFreeSlots(), KEEP_ALIVE,
         TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()));
     setRunningWorker(new ConcurrentHashMap<Long, Worker>());
     try {
       initQueue();
     } catch (DroidsException e) {
-      Core.threadMessage(e.getMessage());
+      for (StackTraceElement message : e.getStackTrace()) {
+        Core.threadMessage(message.getClassName()+"\n"+message.getLineNumber()+": "+
+            message.getMethodName()+"\n"+message.toString());
+      }
     }
     startWorkers();
     Core.threadMessage("Finshed invocation, waiting for workers to finish.");
   }
 
-  private synchronized long startWorkers() {
-    long id=0; 
-    try {
-      Worker worker = getWorker();
-      id = worker.getId();
-      getRunningWorker().put(id, worker);
-      getPool().execute(worker);
-      Core.threadMessage("suspending "+id);
-      Thread.sleep(4000);
-    } catch (Exception e) {
-      Core.threadMessage(e.getMessage());
+  /* 
+   * @see org.apache.droids.AbstractDroid#startWorkers()
+   */
+  public long startWorkers() {
+    synchronized (this) {
+      long id=0; 
+      try {
+        Worker worker = getWorker();
+        id = worker.getId();
+        getRunningWorker().put(id, worker);
+        getPool().execute(worker);
+        Core.threadMessage("suspending "+id);
+        Thread.sleep(THREAD_SLEEP);
+      } catch (InterruptedException e) {
+        Core.threadMessage(e.getMessage());
+      } catch (DroidsException e) {
+        Core.threadMessage(e.getMessage());
+      } 
+      return id;
     }
-    return id;
+    
   }
 
-  /**
+  /*
    * Start the queue.
+   * @see org.apache.droids.api.Droid#initQueue()
    */
-  public synchronized void initQueue() throws DroidsException{
-    QueueLink initialLink = new QueueLink(url, taskDate, 0);
-    getQueue().init((Task[]) new Task[] { initialLink });
-  }
-
-  /**
-   * Get the default worker for the class.
-   * @return
-   */
-  public synchronized Worker getWorker() throws DroidsException {
-    DefaultWorker worker = new DefaultWorker();
-    worker.setQueue(getQueue());
-    worker.setDroid(this);
-    return worker;
-  }
-
-  public synchronized void setQueue(Queue queue) {
-    this.queue = queue;
-  }
-
-  protected synchronized Queue getQueue() {
-    return queue;
-  }
-
-  private Core core;
-
-  /**
-   * Number of currently running threads This value is handed to the threads as
-   * an id, so note that the thread id is not unique, but is always in the range
-   * 0...maxThreads-1
-   */
-  int runningThreads;
-
-  /**
-   * Get number of currently running threads
-   */
-  public int getRunningThreads() {
-    return runningThreads;
-  }
-
-  /**
-   * maximum number of parallel threads -1 if unlimited
-   */
-  int maxThreads;
-
-  /**
-   * Adjust number of allowed threads
-   */
-  public synchronized void setMaxThreads(int maxThreads) {
-    this.maxThreads = maxThreads;
-  }
-
-  /**
-   * Get number of maximum allowed threads
-   */
-  public int getMaxThreads() {
-    return maxThreads;
+  public void initQueue() throws DroidsException{
+    synchronized (this) {
+      QueueLink initialLink = new QueueLink(url, getTaskDate(), 0);
+      getQueue().init(new Task[] { initialLink });
+    }
+    
   }
-
-  protected ThreadPoolExecutor pool;
   
+  private String url = "";
   /**
-   * Shutdown all threads, close the pools and leave.
-   * If it is not working by asking nice to shutdown just kill all
-   * threads.
+   * Return the initial url
+   * @return the initial url
    */
-
-  protected synchronized void shutdownAndAwaitTermination() {
-    Core.threadMessage("SHUTTING DOWN");
-    getPool().shutdown(); // Disable new tasks from being submitted
-    try {
-      // Wait a while for existing tasks to terminate
-      if (!getPool().awaitTermination(1, TimeUnit.SECONDS)) {
-        Core.threadMessage("SHUT DOWN NOW");
-        getPool().shutdownNow(); // Cancel currently executing tasks
-        // Wait a while for tasks to respond to being cancelled
-        if (!getPool().awaitTermination(1, TimeUnit.SECONDS))
-          Core.threadMessage("Pool did not terminate");
-      }
-    } catch (InterruptedException ie) {
-      // (Re-)Cancel if current thread also interrupted
-      getPool().shutdownNow();
-      // Preserve interrupt status
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  private String url;
-
   public String getUrl() {
     return url;
   }
 
+  /**
+   * Set the initial url
+   * @param url the initial url
+   */
   public void setUrl(String url) {
     this.url = url;
   }
 
-  public synchronized void finishedWorker(long id) throws DroidsException {
-    Worker worker = getRunningWorker().get(id);
-    if (null != worker) {
-      int y = worker.getDepth() + 1;
-      getPool().remove(worker);
-      getRunningWorker().remove(id);
-      Core.threadMessage("Worker \"" + id + "\" has finished.");
-      if (getRunningWorker().size() == 0 & !getQueue().hasNext()) {
-        shutdownAndAwaitTermination();
-        Core.threadMessage("All threads has finished.");
-      } else if (getQueue().hasNext()) {
-        for (int i = 0; i < (getMaxThreads() - getPool().getActiveCount()); i++) {
-          if (!getQueue().hasNext())
-            break;
-          startWorkers();
-        }
-      }
+  
+  /* 
+   * @see org.apache.droids.AbstractDroid#getWorker()
+   */
+  public  Worker getWorker() throws DroidsException {
+    DefaultWorker worker = new DefaultWorker();
+    synchronized (this) {
+      worker.setQueue(getQueue());
+      worker.setDroid(this);
+      return worker;
     }
-  }
-
-  public Core getCore() {
-    return core;
-  }
-
-  public void setCore(Core core) {
-    this.core = core;
-  }
-
-  public ThreadPoolExecutor getPool() {
-    return pool;
-  }
-
-  public void setPool(ThreadPoolExecutor pool) {
-    this.pool = pool;
-  }
-
-  public ConcurrentHashMap<Long, Worker> getRunningWorker() {
-    return runningWorker;
-  }
-
-  public void setRunningWorker(ConcurrentHashMap<Long, Worker> runningWorker) {
-    this.runningWorker = runningWorker;
-  }
-
-  public void setRunningThreads(int runningThreads) {
-    this.runningThreads = runningThreads;
-  }
-
-  public String getTaskDate() {
-    return taskDate;
-  }
-
-  public void setTaskDate(String taskDate) {
-    this.taskDate = taskDate;
-  }
-
-  public int getFreeSlots() {
-    return freeSlots;
-  }
-
-  public void setFreeSlots(int freeSlots) {
-    this.freeSlots = freeSlots;
+    
   }
 
 }

Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java Thu Jul 24 02:43:57 2008
@@ -98,30 +98,35 @@
 
   protected void filter(Parse parse) {
     Outlink[] filterLinks = filterLinks(parse);
-    queue.merge(filterLinks);
+    getQueue().merge(filterLinks);
   }
 
   protected Outlink[] filterLinks(Parse parse) {
-    // filter the link
-    Outlink[] links = parse.getData().getOutlinks();
-    // new cleaned list
-    ArrayList<Outlink> filtered = new ArrayList<Outlink>();
-    for (int i = 0; i < links.length; i++) {
-      Outlink outlink = links[i];
-      if (filtersFactory.accept(outlink.getToUrl()) & !filtered.contains(outlink)) {
-        filtered.add(outlink);
+    synchronized (this) {
+   // filter the link
+      Outlink[] links = parse.getData().getOutlinks();
+      // new cleaned list
+      ArrayList<Outlink> filtered = new ArrayList<Outlink>();
+      for (int i = 0; i < links.length; i++) {
+        Outlink outlink = links[i];
+        if (filtersFactory.accept(outlink.getToUrl()) && !filtered.contains(outlink)) {
+          filtered.add(outlink);
+        }
       }
+      // this are the links we need to follow
+      Outlink[] filterLinks = filtered.toArray(new Outlink[filtered.size()]);
+      return filterLinks;
     }
-    // this are the links we need to follow
-    Outlink[] filterLinks = filtered.toArray(new Outlink[filtered.size()]);
-    return filterLinks;
   }
 
   public void setQueue(Queue queue) {
-    this.queue = queue;
-    link = queue.next();
-    if (link != null)
-      depth = link.getDepth();
+    synchronized (this) {
+      this.queue = queue;
+      link = getQueue().next();
+      if (link != null) {
+        depth = link.getDepth();
+      }
+    }
   }
 
   public void setDroid(Droid droid) throws DroidsException {
@@ -164,24 +169,40 @@
     return handlerFactory;
   }
 
-  public synchronized Droid getDroid() {
-    return droid;
+  public Droid getDroid() {
+    synchronized (this) {
+      return droid;
+    }
+  }
+
+  public void setUri(String uri) {
+    synchronized (this) {
+      this.uri = uri;
+    }
   }
 
-  public synchronized void setUri(String uri) {
-    this.uri = uri;
+  public String getUri() {
+    synchronized (this) {
+      return uri;
+    }
   }
 
-  public synchronized String getUri() {
-    return uri;
+  public void setProtocol(Protocol protocol) {
+    synchronized (this) {
+      this.protocol = protocol;
+    }
   }
 
-  public synchronized void setProtocol(Protocol protocol) {
-    this.protocol = protocol;
+  public Protocol getProtocol() {
+    synchronized (this) {
+      return protocol;
+    }
   }
 
-  public synchronized Protocol getProtocol() {
-    return protocol;
+  public Queue getQueue() {
+    synchronized (this) {
+      return queue;
+    }
   }
 
 }
\ No newline at end of file

Modified: labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml Thu Jul 24 02:43:57 2008
@@ -53,9 +53,9 @@
     <property name="file" value="@droids.filter.regex@"/>
   </bean>
   <!-- Handler -->
-  <bean 
+  <!--<bean 
     name="org.apache.droids.api.Handler/org.apache.droids.handle.Sysout"
-    class="org.apache.droids.handle.Sysout"/>
+    class="org.apache.droids.handle.Sysout"/>-->
   <bean 
     name="org.apache.droids.api.Handler/org.apache.droids.handle.Save"
     class="org.apache.droids.handle.Save">

Modified: labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/HandlerFactory.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/HandlerFactory.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/HandlerFactory.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/HandlerFactory.java Thu Jul 24 02:43:57 2008
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.util.Iterator;
 
 import org.apache.droids.api.Handler;
 import org.apache.droids.api.Parse;
@@ -53,7 +54,9 @@
     } catch (IOException e) {
       log.fatal("Handler setup has thrown an error.", e);
     }
-    for (String handlerName : (String[]) getMap().keySet().toArray()) {
+    for (Iterator<String> iterator = getMap().keySet().iterator(); iterator
+    .hasNext();) {
+      String handlerName = iterator.next();
       if (streamCopy == null) {
         return false;
       }

Modified: labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/URLFiltersFactory.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/URLFiltersFactory.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/URLFiltersFactory.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/helper/factories/URLFiltersFactory.java Thu Jul 24 02:43:57 2008
@@ -16,6 +16,8 @@
  */
 package org.apache.droids.helper.factories;
 
+import java.util.Iterator;
+
 import org.apache.droids.api.URLFilter;
 
 /**
@@ -35,7 +37,9 @@
    * @return true if filter plugin accept the url, false if excluded.
    */
   public boolean accept(String urlString) {
-    for (String key : (String[]) getMap().keySet().toArray()) {
+    for (Iterator<String> iterator = getMap().keySet().iterator(); iterator
+    .hasNext();) {
+      String key = iterator.next();
       if (urlString == null) {
         return false;
       }

Modified: labs/droids/trunk/src/core/java/org/apache/droids/parse/html/HtmlParser.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/parse/html/HtmlParser.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/parse/html/HtmlParser.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/parse/html/HtmlParser.java Thu Jul 24 02:43:57 2008
@@ -136,6 +136,9 @@
               final Outlink outlink = new Outlink(
                   target.contains(":/") ? target : new URL(base, target)
                       .toString(), link.getDepth() + 1);
+              System.out.println("set size: "+set.size());
+              System.out.println("outlink.getToUrl(): "+outlink.getToUrl());
+              System.out.println("set.contains(outlink.getToUrl(): "+set.contains(outlink.getToUrl()));
               if (!set.contains(outlink.getToUrl())) {
                 set.add(outlink.getToUrl());
                 links.add(outlink);

Modified: labs/droids/trunk/src/core/java/org/apache/droids/queue/QueueBean.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/queue/QueueBean.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/queue/QueueBean.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/queue/QueueBean.java Thu Jul 24 02:43:57 2008
@@ -82,7 +82,10 @@
    * @return all open queue items that have not finished.
    */
   public Task[] getToDoTasks() {
-    return toDoLinks.clone();
+    if (null != toDoLinks){
+      return toDoLinks.clone();
+    }
+    return null;
   }
 
   /**
@@ -171,6 +174,8 @@
    *                an array of all tasks that we need to add to the queue.
    */
   public void setToDoLinks(Task[] toDoLinks) {
-    this.toDoLinks = toDoLinks.clone();
+    if(null!=toDoLinks){
+      this.toDoLinks = toDoLinks.clone();      
+    }
   }
 }
\ No newline at end of file

Modified: labs/droids/trunk/src/core/java/org/apache/droids/queue/Simple.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/queue/Simple.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/queue/Simple.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/queue/Simple.java Thu Jul 24 02:43:57 2008
@@ -35,7 +35,7 @@
    * @see org.apache.droids.api.Queue#getTask(java.lang.String)
    */
   public Task getTask(String id) {
-    synchronized (allTasks) {
+    synchronized (this) {
       return allTasks.get(id);
     }
 
@@ -45,19 +45,17 @@
    * @see org.apache.droids.api.Queue#totalSize()
    */
   public int totalSize() {
-    synchronized (allTasks) {
+    synchronized (this) {
       return allTasks.size();
     }
   }
 
   public Simple() {
-    synchronized (allTasks) {
-      allTasks = new LinkedHashMap<String, Task>();
-    }
+    allTasks = new LinkedHashMap<String, Task>();
   }
 
   public void init(Task[] initialTask) {
-    synchronized (allTasks) {
+    synchronized (this) {
       LinkedList<Task> list = new LinkedList<Task>();
       for (int i = 0; i < initialTask.length; i++) {
         Link task = (Link) initialTask[i];
@@ -67,7 +65,6 @@
         }
       }
       this.setToDoLinks(list.toArray(new Link[list.size()]));
-
     }
   }
 
@@ -82,8 +79,9 @@
   }
 
   public void merge(Task[] filterLinks) {
-    synchronized (allTasks) {
+    synchronized (this) {
       LinkedList<Task> list = new LinkedList<Task>();
+      System.out.println("getToDoTasks().length " +getToDoTasks().length);
       if (null != getToDoTasks()) {
         for (int i = 0; i < getToDoTasks().length; i++) {
           Task task = getToDoTasks()[i];
@@ -95,6 +93,7 @@
       }
       for (int i = 0; i < filterLinks.length; i++) {
         Task task = filterLinks[i];
+        System.out.println("allTasks.size() "+allTasks.size());
         if (null != task && acceptSize(i + allTasks.size())
             && !allTasks.containsKey(task.getId())
             && acceptDepth(task.getDepth())) {
@@ -108,7 +107,7 @@
   }
 
   public Task next() {
-    synchronized (getToDoTasks()) {
+    synchronized (this) {
       if (getToDoTasks() == null || getToDoTasks().length < 1) {
         return null;
       }

Modified: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java (original)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerCrawler.java Thu Jul 24 02:43:57 2008
@@ -7,6 +7,7 @@
 import org.apache.droids.Core;
 import org.apache.droids.DefaultCrawler;
 import org.apache.droids.api.Worker;
+import org.apache.droids.exception.DroidsException;
 import org.apache.droids.handle.Solr;
 import org.apache.http.PostFile;
 
@@ -17,8 +18,9 @@
    * We want to use the indexerWorker since it limits
    * the handling to the SolrHandler
    * @return IndexerWorker
+   * @throws DroidsException 
    */
-  public synchronized Worker getWorker() {
+  public synchronized Worker getWorker() throws DroidsException {
     IndexerWorker worker = new IndexerWorker();
     worker.setQueue(getQueue());
     worker.setDroid(this);

Modified: labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java?rev=679332&r1=679331&r2=679332&view=diff
==============================================================================
--- labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java (original)
+++ labs/droids/trunk/src/examples/java/org/apache/droids/examples/IndexerWorker.java Thu Jul 24 02:43:57 2008
@@ -8,6 +8,7 @@
 import org.apache.droids.DefaultWorker;
 import org.apache.droids.api.Handler;
 import org.apache.droids.api.Parse;
+import org.apache.droids.exception.DroidsException;
 
 public class IndexerWorker extends DefaultWorker {
   protected void handle(Parse parse) throws MalformedURLException, IOException {
@@ -17,6 +18,11 @@
     } catch (Exception e) {
       Core.threadMessage(e.getMessage());
     }
-    getDroid().finishedWorker(super.getId());
+    try {
+      getDroid().finishedWorker(super.getId());
+    } catch (DroidsException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org