incubator-droids-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1429788 - in /incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids: core/ exception/ handle/ helper/factories/
Date Mon, 07 Jan 2013 13:07:00 GMT
Author: tobr
Date: Mon Jan  7 13:07:00 2013
New Revision: 1429788

URL: http://svn.apache.org/viewvc?rev=1429788&view=rev
Log:
removed Protocol
added Fetcher

Added:
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
      - copied, changed from r1429330, incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
Removed:
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/exception/ProtocolNotFoundException.java
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java
Modified:
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
    incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java

Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
Mon Jan  7 13:07:00 2013
@@ -19,7 +19,6 @@ package org.apache.droids.core;
 import org.apache.droids.helper.factories.FilterFactory;
 import org.apache.droids.helper.factories.HandlerFactory;
 import org.apache.droids.helper.factories.ParserFactory;
-import org.apache.droids.helper.factories.ProtocolFactory;
 import org.apache.droids.filter.Filter;
 import org.apache.droids.parse.Parser;
 import org.slf4j.Logger;
@@ -35,7 +34,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class AbstractDroid<T extends Task> implements Droid<T> {
     protected final Queue<T> queue;
     protected final TaskMaster<T> taskMaster;
-    protected ProtocolFactory protocolFactory;
+    protected Fetcher fetcher;
     protected ParserFactory parserFactory;
     protected FilterFactory filterFactory;
     protected HandlerFactory handlerFactory;
@@ -49,7 +48,6 @@ public abstract class AbstractDroid<T ex
     public AbstractDroid(Queue<T> queue, TaskMaster<T> taskMaster) {
         this.queue = queue;
         this.taskMaster = taskMaster;
-        this.protocolFactory = new ProtocolFactory();
         this.parserFactory = new ParserFactory();
         this.filterFactory = new FilterFactory();
         this.handlerFactory = new HandlerFactory();
@@ -66,11 +64,26 @@ public abstract class AbstractDroid<T ex
     }
 
     @Override
+    public void finished() {
+        logger.info("FINISHED!!!");
+    }
+
+
+    @Override
     public void add(T task) {
         queue.add(task);
     }
 
     @Override
+    public void load(T task) throws DroidsException, IOException {
+        if (this.fetcher == null) {
+            throw new DroidsException("Fetcher not set");
+        } else {
+            this.fetcher.fetch(task);
+        }
+    }
+
+    @Override
     public void parse(T task) throws DroidsException, IOException {
         this.parserFactory.parse(task);
     }
@@ -85,18 +98,16 @@ public abstract class AbstractDroid<T ex
         return this.filterFactory.filter(task);
     }
 
+    public void setFetcher(Fetcher fetcher) {
+        this.fetcher = fetcher;
+    }
+
     public void addParsers(Parser... parsers) {
         for (Parser parser : parsers) {
             this.parserFactory.addParser(parser);
         }
     }
 
-    public void addProtocols(Protocol... protocols) {
-        for (Protocol protocol : protocols) {
-            this.protocolFactory.addProtocol(protocol);
-        }
-    }
-
     public void addHandlers(Handler... handlers) {
         for (Handler handler : handlers) {
             this.handlerFactory.addHandler(handler);

Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
Mon Jan  7 13:07:00 2013
@@ -1,5 +1,6 @@
 package org.apache.droids.core;
 
+import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -14,6 +15,7 @@ public class ContentEntity {
     public final static String CONTENT = "content";
     public final static String MIME_TYPE = "mime";
     public final static String CONTENT_LENGTH = "content-length";
+    public final static String LINKS = "links";
 
     public ContentEntity() {
         this.data = new HashMap<String, Object>();
@@ -27,6 +29,22 @@ public class ContentEntity {
         this.data.put(key, value);
     }
 
+    public void setContent(InputStream in) {
+        this.put(CONTENT, in);
+    }
+
+    public InputStream getContent() throws DroidsException {
+        if (this.getValue(CONTENT) != null) {
+            if (this.getValue(CONTENT) instanceof InputStream) {
+                return (InputStream)this.getValue(CONTENT);
+            } else {
+                throw new DroidsException("wrong type of content");
+            }
+        } else {
+            throw new NullPointerException();
+        }
+    }
+
     public Map<String, Object> getData() {
         return data;
     }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
Mon Jan  7 13:07:00 2013
@@ -34,7 +34,8 @@ import java.io.IOException;
 public interface Droid<T extends Task> {
 
     /**
-     * Invoke an instance of the worker used in the droid
+     * Start the droid with checking the preconditions and
+     * initializing the taskmaster with the initial tasks
      */
     public void start();
 
@@ -56,23 +57,30 @@ public interface Droid<T extends Task> {
     public Worker<T> getNewWorker();
 
     /**
-     * Parse the task
+     * Load the content of the task using the fetcher.
      *
-     * @param task the task
+     * @param task the task for loading the content
+     */
+    public void load(T task) throws DroidsException, IOException;
+
+    /**
+     * Parse the task with the defined Parsers.
+     *
+     * @param task the task to parse
      */
     public void parse(T task) throws DroidsException, IOException;
 
     /**
      * Handle the task.
      *
-     * @param task
+     * @param task the task to handle
      */
     public void handle(T task) throws DroidsException, IOException;
 
     /**
      * Filter the task.
      *
-     * @param task
+     * @param task the task to filter
      * @return the task or null, if the task is not valid
      */
     public T filter(T task);

Copied: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
(from r1429330, incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java)
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java?p2=incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java&p1=incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java&r1=1429330&r2=1429788&rev=1429788&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
Mon Jan  7 13:07:00 2013
@@ -17,7 +17,6 @@
 package org.apache.droids.core;
 
 import java.io.IOException;
-import java.net.URI;
 
 
 /**
@@ -26,25 +25,23 @@ import java.net.URI;
  *
  * @version 1.0
  */
-public interface Protocol {
+public interface Fetcher<T extends Task> {
     /**
      * Some protocols (like http) offer a mechanism to evaluate whether the client
      * can request a given url (in http this is the robots.txt configuration)
      *
-     * @param url the url to evaluate
-     * @return true if we can request the url. false if we are forbidden.
-     * @throws MalformedURLException
+     * @param task the task to evaluate
+     * @return true if we can request the task. false if we are forbidden.
+     * @throws IOException
      */
-    boolean isAllowed(URI url) throws IOException;
+    public boolean isAllowed(T task) throws IOException;
 
     /**
      * Return the content entity represent of the url
      *
-     * @param url url of the stream we want to open
-     * @return the content of the given url
+     * @param task the task we want to retrieve.
      * @throws IOException
      */
-    Task load(URI uri) throws IOException;
+    public void fetch(T task) throws IOException;
 
-    String scheme();
 }
\ No newline at end of file

Modified: incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
Mon Jan  7 13:07:00 2013
@@ -23,6 +23,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.util.Set;
 
 /**
  * Handler that write the stream to the sysout. Mostly added for debugging
@@ -31,8 +32,18 @@ import java.io.*;
  * @version 1.0
  */
 public class SysoutHandler extends WriterHandler implements Handler {
+    private Set<String> attributes;
     private static final Logger logger = LoggerFactory.getLogger(SysoutHandler.class);
 
+    public SysoutHandler() {
+        super();
+    }
+
+    public SysoutHandler(Set<String> attributes) {
+        super();
+        this.attributes = attributes;
+    }
+
     private static void writeOutput(InputStream stream) throws IOException {
         Reader reader = new InputStreamReader(stream);
         Writer output = new OutputStreamWriter(System.out);
@@ -42,16 +53,18 @@ public class SysoutHandler extends Write
     @Override
     public void handle(Task task) throws IOException, DroidsException {
         for (String key : task.getContentEntity().getData().keySet()) {
-            if (task.getContentEntity().getValue(key) instanceof InputStream) {
-                InputStream instream = (InputStream) task.getContentEntity().getValue(key);
-                try {
-                    logger.info(key + ": ");
-                    writeOutput(instream);
-                } finally {
-                    instream.close();
+            if (attributes.contains(key)) {
+                if (task.getContentEntity().getValue(key) instanceof InputStream) {
+                    InputStream instream = (InputStream) task.getContentEntity().getValue(key);
+                    try {
+                        logger.info(key + ": ");
+                        writeOutput(instream);
+                    } finally {
+                        instream.close();
+                    }
+                } else {
+                    logger.info(key + ": " + task.getContentEntity().getValue(key).toString());
                 }
-            } else {
-                logger.info(key + ": " + task.getContentEntity().getValue(key).toString());
             }
         }
     }



Mime
View raw message