incubator-droids-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1445250 - in /incubator/droids/branches/0.2.x-cleanup/droids-crawler/src: main/java/org/apache/droids/crawler/ main/java/org/apache/droids/fetcher/ test/java/org/apache/droids/crawler/
Date Tue, 12 Feb 2013 16:38:14 GMT
Author: tobr
Date: Tue Feb 12 16:38:13 2013
New Revision: 1445250

URL: http://svn.apache.org/r1445250
Log:
updated crawler module  to support the new ParserData API

Modified:
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlerTask.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/fetcher/CrawlingFetcher.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/crawler/CrawlingDroidTest.java

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlerTask.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlerTask.java?rev=1445250&r1=1445249&r2=1445250&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlerTask.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlerTask.java
Tue Feb 12 16:38:13 2013
@@ -18,10 +18,11 @@ package org.apache.droids.crawler;
 
 import org.apache.droids.core.ContentEntity;
 import org.apache.droids.core.LinkedTask;
+import org.apache.droids.core.ParserData;
 import org.apache.droids.core.Task;
+import org.apache.droids.parse.LinkedParserData;
 
 import java.net.URI;
-import java.util.Collection;
 import java.util.Date;
 import java.io.Serializable;
 
@@ -30,15 +31,15 @@ import java.io.Serializable;
  * Basic implementation for Crawler @Task. LinkTasks are working instructions for URI
  * based droids.
  */
-public class CrawlerTask implements LinkedTask, Serializable {
+public class CrawlerTask extends LinkedTask implements Serializable {
     private static final long serialVersionUID = -44808094386453088L;
 
     private ContentEntity contentEntity;
+    private LinkedParserData parserData;
     private Date started;
     private final int depth;
     private final URI uri;
     private final LinkedTask from;
-    private Collection<LinkedTask> linksTo;
     private boolean aborted = false;
 
     /**
@@ -53,20 +54,6 @@ public class CrawlerTask implements Link
     /**
      * Creates a new LinkTask.
      *
-     * @param uri The URI of the task.
-     * @param depth The depth of the task.
-     */
-    public CrawlerTask(LinkedTask from, URI uri, int depth) {
-        this.from = from;
-        this.uri = uri;
-        this.depth = depth;
-        this.started = new Date();
-        this.contentEntity = new ContentEntity();
-    }
-
-    /**
-     * Creates a new LinkTask.
-     *
      * @param from Link
      * @param uri URI
      * @param depth int
@@ -74,10 +61,24 @@ public class CrawlerTask implements Link
      */
     public CrawlerTask(LinkedTask from, URI uri, int depth, String anchorText) {
         this(from, uri, depth);
+        this.parserData.setAnchorText(anchorText);
+    }
+
+    /**
+     * Creates a new LinkTask.
+     *
+     * @param uri The URI of the task.
+     * @param depth The depth of the task.
+     */
+    public CrawlerTask(LinkedTask from, URI uri, int depth) {
         this.contentEntity = new ContentEntity();
-        this.contentEntity.setAnchorText(anchorText);
+        this.parserData = new LinkedParserData();
+        this.from = from;
+        this.uri = uri;
+        this.depth = depth;
     }
 
+
     @Override
     public URI getURI() {
         return uri;
@@ -89,6 +90,11 @@ public class CrawlerTask implements Link
     }
 
     @Override
+    public ParserData getParserData() {
+        return this.parserData;
+    }
+
+    @Override
     public Date getTaskDate() {
         return started;
     }
@@ -123,18 +129,5 @@ public class CrawlerTask implements Link
         return from;
     }
 
-    @Override
-    public Collection<LinkedTask> getTo() {
-        return linksTo;
-    }
-
-    /**
-     * Set Outgoing links.
-     *
-     * @param linksTo Collection<URI>
-     */
-    public void setLinksTo(Collection<LinkedTask> linksTo) {
-        this.linksTo = linksTo;
-    }
 
-}
\ No newline at end of file
+}

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java?rev=1445250&r1=1445249&r2=1445250&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
Tue Feb 12 16:38:13 2013
@@ -22,7 +22,7 @@ import org.apache.droids.core.Worker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Set;
+import java.util.Collection;
 
 public class CrawlingWorker implements Worker<LinkedTask> {
 
@@ -43,7 +43,7 @@ public class CrawlingWorker implements W
             droid.load(task);
             droid.parse(task);
             // add this to a link handler
-            Set<LinkedTask> links = task.getContentEntity().getLinks();
+            Collection<LinkedTask> links = task.getTo();
             if (links != null) {
                 for (LinkedTask outLink : links) {
                     LinkedTask filteredTask = droid.filter(outLink);

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/fetcher/CrawlingFetcher.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/fetcher/CrawlingFetcher.java?rev=1445250&r1=1445249&r2=1445250&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/fetcher/CrawlingFetcher.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/fetcher/CrawlingFetcher.java
Tue Feb 12 16:38:13 2013
@@ -1,5 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.droids.fetcher;
 
+import org.apache.droids.core.ContentEntity;
 import org.apache.droids.core.Fetcher;
 import org.apache.droids.core.LinkedTask;
 import org.apache.droids.norobots.ContentLoader;
@@ -20,6 +37,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  *
@@ -31,7 +50,7 @@ public class CrawlingFetcher implements 
     private String userAgent;
     private final HttpClient httpClient;
     private final ContentLoader contentLoader;
-    private final static String DROIDS_USER_AGENT = "Apache-Droids/0.3 (java 1.5)";
+    private final static String DROIDS_USER_AGENT = "Apache-Droids/0.3 (java 1.6)";
     private final static Logger logger = LoggerFactory.getLogger(CrawlingFetcher.class);
 
     public CrawlingFetcher() {
@@ -87,10 +106,12 @@ public class CrawlingFetcher implements 
         }
         HttpEntity entity = response.getEntity();
         if (entity != null) {
+            ContentEntity contentEntity = task.getContentEntity();
             InputStream instream = entity.getContent();
-            task.getContentEntity().setContent(instream);
+            contentEntity.setContent(instream);
+            Map<String, String> headerData = new HashMap<String, String>();
             for (Header header : response.getAllHeaders()) {
-                task.getContentEntity().put(header.getName(), header.getValue());
+                contentEntity.set(header.getName(), header.getValue());
             }
         }
     }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/crawler/CrawlingDroidTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/crawler/CrawlingDroidTest.java?rev=1445250&r1=1445249&r2=1445250&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/crawler/CrawlingDroidTest.java
(original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/crawler/CrawlingDroidTest.java
Tue Feb 12 16:38:13 2013
@@ -53,7 +53,7 @@ public class CrawlingDroidTest {
 
         SimpleCrawlingDroid droid = new SimpleCrawlingDroid(queue, taskMaster);
         droid.setInitialLocations(initialLocations);
-        droid.addParsers(new SimpleLinkParser<LinkedTask>());
+        droid.addParsers(new SimpleLinkParser());
 
         ReportHandler reportHandler = new ReportHandler();
         droid.addHandlers(reportHandler);



Mime
View raw message