incubator-droids-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1423339 [1/5] - in /incubator/droids/branches/0.2.x-cleanup: droids-crawler/ droids-crawler/src/main/java/org/apache/droids/crawler/ droids-crawler/src/main/java/org/apache/droids/protocol/http/ droids-crawler/src/test/java/org/apache/droi...
Date Tue, 18 Dec 2012 08:48:00 GMT
Author: tobr
Date: Tue Dec 18 08:47:39 2012
New Revision: 1423339

URL: http://svn.apache.org/viewvc?rev=1423339&view=rev
Log:
reformat code base

Modified:
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/DroidsFactory.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/ExceptionCrawlingDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/FileRenameDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/SaveCrawlingDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/SysoutCrawlingDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/cli/SimpleRuntime.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/localserver/LocalHttpServer.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/main/java/org/apache/droids/localserver/ResourceHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/test/java/org/apache/droids/examples/TestCrawlingWorker.java
    incubator/droids/branches/0.2.x-cleanup/droids-examples/src/test/java/org/apache/droids/examples/TestSimpleDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/AbstractRule.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/AllowedRule.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/ContentLoader.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/DisallowedRule.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/NoRobotException.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/Rule.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/RulesEngine.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/SimpleContentLoader.java
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/main/java/org/apache/droids/norobots/package.html
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/site/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-norobots/src/test/java/org/apache/droids/norobots/TestNorobotsClient.java
    incubator/droids/branches/0.2.x-cleanup/droids-solr/example/conf/schema.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/example/conf/solrconfig.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/locationmap.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/index.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/content/xdocs/tabs.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/documentation/skinconf.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/main/java/org/apache/droids/solr/AdvancedSolrHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/main/java/org/apache/droids/solr/SolrHandler.java
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/site/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/AdvancedSolrHandleTest.java
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/MockContentEntity.java
    incubator/droids/branches/0.2.x-cleanup/droids-solr/src/test/java/org/apache/droids/solr/SolrHandleTest.java
    incubator/droids/branches/0.2.x-cleanup/droids-spring/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/locationmap.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/index.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/content/xdocs/tabs.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/documentation/skinconf.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/java/org/apache/droids/dynamic/Cli.java
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/java/org/apache/droids/dynamic/DroidsConfig.java
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-context.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/main/resources/org/apache/droids/dynamic/droids-core-factories-context.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/site/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/java/org/apache/droids/dynamic/DroidsConfigTest.java
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/java/org/apache/droids/dynamic/TestSimpleDroid.java
    incubator/droids/branches/0.2.x-cleanup/droids-spring/src/test/resources/droids-core-test-context.xml
    incubator/droids/branches/0.2.x-cleanup/droids-tika/pom.xml
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/TikaDocumentParser.java
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/TikaHtmlParser.java
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/api/TikaParse.java
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/main/java/org/apache/droids/tika/parse/TikaParseImpl.java
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/site/site.xml
    incubator/droids/branches/0.2.x-cleanup/droids-tika/src/test/java/org/apache/droids/tika/TikaHtmlParserTest.java

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/pom.xml Tue Dec 18 08:47:39 2012
@@ -1,25 +1,26 @@
 <?xml version="1.0"?>
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.droids</groupId>
-    <artifactId>droids</artifactId>
-    <version>0.3.0-incubating-SNAPSHOT</version>
-  </parent>
-  <artifactId>droids-crawler</artifactId>
-  <name>droids-crawler</name>
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.droids</groupId>
-      <artifactId>droids-core</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>3.8.1</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.droids</groupId>
+        <artifactId>droids</artifactId>
+        <version>0.3.0-incubating-SNAPSHOT</version>
+    </parent>
+    <artifactId>droids-crawler</artifactId>
+    <name>droids-crawler</name>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.droids</groupId>
+            <artifactId>droids-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>3.8.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingDroid.java Tue Dec 18 08:47:39 2012
@@ -21,6 +21,7 @@ import java.net.URISyntaxException;
 import java.util.Collection;
 
 import com.google.common.base.Preconditions;
+
 import java.util.Queue;
 
 import org.apache.droids.core.AbstractDroid;
@@ -30,42 +31,42 @@ import org.apache.droids.exception.Inval
 
 public abstract class CrawlingDroid extends AbstractDroid<Link> {
 
-	private Collection<String> initialLocations;
+    private Collection<String> initialLocations;
 
-	public CrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
-		super(queue, taskMaster);
-	}
-
-	public void setInitialLocations(Collection<String> initialLocations) {
-		this.initialLocations = initialLocations;
-	}
-
-	@Override
-	public void init() throws InvalidTaskException {
-		Preconditions.checkState(initialLocations != null,
-				"WebCrawlerDroid requires at least one starting file");
-		Preconditions.checkState(!initialLocations.isEmpty(),
-				"WebCrawlerDroid requires at least one starting file");
-		for (String location : initialLocations) {
-			URI uri;
-			try {
-				uri = new URI(location);
-			} catch (URISyntaxException ex) {
-				throw new InvalidTaskException("Invalid lication: " + location);
-			}
-			queue.offer(new LinkTask(null, uri, 0));
-		}
-	}
-
-	public void start() {
-		taskMaster.start(queue, this);
-	}
-
-	@Override
-	public void finished() {
-		logger.info("FINISHED!!!");
-	}
+    public CrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
+        super(queue, taskMaster);
+    }
+
+    public void setInitialLocations(Collection<String> initialLocations) {
+        this.initialLocations = initialLocations;
+    }
+
+    @Override
+    public void init() throws InvalidTaskException {
+        Preconditions.checkState(initialLocations != null,
+                "WebCrawlerDroid requires at least one starting file");
+        Preconditions.checkState(!initialLocations.isEmpty(),
+                "WebCrawlerDroid requires at least one starting file");
+        for (String location : initialLocations) {
+            URI uri;
+            try {
+                uri = new URI(location);
+            } catch (URISyntaxException ex) {
+                throw new InvalidTaskException("Invalid lication: " + location);
+            }
+            queue.offer(new LinkTask(null, uri, 0));
+        }
+    }
+
+    public void start() {
+        taskMaster.start(queue, this);
+    }
+
+    @Override
+    public void finished() {
+        logger.info("FINISHED!!!");
+    }
 
-	public abstract Worker<Link> getNewWorker();
+    public abstract Worker<Link> getNewWorker();
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/CrawlingWorker.java Tue Dec 18 08:47:39 2012
@@ -35,104 +35,104 @@ import org.slf4j.LoggerFactory;
 
 public class CrawlingWorker implements Worker<Link> {
 
-	private static final Logger LOG = LoggerFactory.getLogger(CrawlingWorker.class);
+    private static final Logger LOG = LoggerFactory.getLogger(CrawlingWorker.class);
 
-	private final CrawlingDroid droid;
-	HandlerFactory handlerFactory;
+    private final CrawlingDroid droid;
+    HandlerFactory handlerFactory;
 
-	public CrawlingWorker(CrawlingDroid droid) {
-		this.droid = droid;
-	}
-
-	@Override
-	public void execute(Link link) throws DroidsException, IOException {
-		final String userAgent = this.getClass().getCanonicalName();
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Starting " + userAgent);
-		}
-		URI uri = link.getURI();
-		final Protocol protocol = droid.getProtocolFactory().getProtocol(uri);
-		if (protocol == null) {
-			if (LOG.isWarnEnabled()) {
-				LOG.warn("Unsupported protocol scheme '" + uri.getScheme() + "'");
-			}
-			return;
-		}
-
-		if (protocol.isAllowed(uri)) {
-			if (LOG.isInfoEnabled()) {
-				LOG.info("Loading " + uri);
-			}
+    public CrawlingWorker(CrawlingDroid droid) {
+        this.droid = droid;
+    }
+
+    @Override
+    public void execute(Link link) throws DroidsException, IOException {
+        final String userAgent = this.getClass().getCanonicalName();
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Starting " + userAgent);
+        }
+        URI uri = link.getURI();
+        final Protocol protocol = droid.getProtocolFactory().getProtocol(uri);
+        if (protocol == null) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("Unsupported protocol scheme '" + uri.getScheme() + "'");
+            }
+            return;
+        }
+
+        if (protocol.isAllowed(uri)) {
+            if (LOG.isInfoEnabled()) {
+                LOG.info("Loading " + uri);
+            }
 //			ContentEntity entity = null;
-			try {
+            try {
 //				entity = protocol.load(uri);
-			} catch (OutOfMemoryError e) {
-				LOG.error("Out of memory processing: " + uri + " skipping", e);
-				throw new DroidsException(e);
-			}
-			try {
+            } catch (OutOfMemoryError e) {
+                LOG.error("Out of memory processing: " + uri + " skipping", e);
+                throw new DroidsException(e);
+            }
+            try {
 //				String contentType = entity.getMimeType();
-				String contentType = "";
-				if (LOG.isDebugEnabled()) {
-					LOG.debug("Content type " + contentType);
-				}
-				if (contentType == null) {
-					LOG.info("Missing content type... can't parse...");
-				} else {
-					Parser parser = droid.getParserFactory().getParser(contentType);
-					if (parser == null) {
-						if (LOG.isDebugEnabled()) {
-							LOG.debug("Could not find parser for " + contentType);
-						}
-					} else {
+                String contentType = "";
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("Content type " + contentType);
+                }
+                if (contentType == null) {
+                    LOG.info("Missing content type... can't parse...");
+                } else {
+                    Parser parser = droid.getParserFactory().getParser(contentType);
+                    if (parser == null) {
+                        if (LOG.isDebugEnabled()) {
+                            LOG.debug("Could not find parser for " + contentType);
+                        }
+                    } else {
 //						Parse parse = parser.parse(entity, link);
-						Parse parse = null;
-						if (parse.getNewTasks() != null && parse.isFollowed()) {
-							Collection<Link> outlinks = getFilteredOutlinks(parse);
-							droid.getQueue().addAll(outlinks);
-						}
+                        Parse parse = null;
+                        if (parse.getNewTasks() != null && parse.isFollowed()) {
+                            Collection<Link> outlinks = getFilteredOutlinks(parse);
+                            droid.getQueue().addAll(outlinks);
+                        }
 //						entity.setParse(parse);
-						handle(link);
-					}
-				}
-			} finally {
+                        handle(link);
+                    }
+                }
+            } finally {
 //				entity.finish();
-			}
-		} else {
-			if (LOG.isInfoEnabled()) {
-				LOG.info("Stopping processing since" + " bots are not allowed for " + uri);
-			}
-		}
-	}
-
-	protected void handle(Task task) throws DroidsException, IOException {
-		getHandlerFactory().handle(task);
-	}
-
-	protected Collection<Link> getFilteredOutlinks(Parse parse) {
-		URLFiltersFactory filters = droid.getFiltersFactory();
-
-		// TODO -- make the hashvalue for Outlink...
-		Map<URI, Link> filtered = new LinkedHashMap<URI, Link>();
-		for (Task outTask : parse.getNewTasks()) {
-			// only use Links, so if for some reason it isn't a Link, skip
-			if (!(outTask instanceof Link)) {
-				continue;
-			}
-			Link outlink = (Link) outTask;
-			URI uri = outlink.getURI();
-			if (filters.accept(outlink) && !filtered.containsKey(uri)) {
-					filtered.put(uri, outlink);
-			}
-		}
-		return filtered.values();
-	}
-
-	public HandlerFactory getHandlerFactory() {
-		return handlerFactory;
-	}
-
-	public void setHandlerFactory(HandlerFactory handlerFactory) {
-		this.handlerFactory = handlerFactory;
-	}
+            }
+        } else {
+            if (LOG.isInfoEnabled()) {
+                LOG.info("Stopping processing since" + " bots are not allowed for " + uri);
+            }
+        }
+    }
+
+    protected void handle(Task task) throws DroidsException, IOException {
+        getHandlerFactory().handle(task);
+    }
+
+    protected Collection<Link> getFilteredOutlinks(Parse parse) {
+        URLFiltersFactory filters = droid.getFiltersFactory();
+
+        // TODO -- make the hashvalue for Outlink...
+        Map<URI, Link> filtered = new LinkedHashMap<URI, Link>();
+        for (Task outTask : parse.getNewTasks()) {
+            // only use Links, so if for some reason it isn't a Link, skip
+            if (!(outTask instanceof Link)) {
+                continue;
+            }
+            Link outlink = (Link) outTask;
+            URI uri = outlink.getURI();
+            if (filters.accept(outlink) && !filtered.containsKey(uri)) {
+                filtered.put(uri, outlink);
+            }
+        }
+        return filtered.values();
+    }
+
+    public HandlerFactory getHandlerFactory() {
+        return handlerFactory;
+    }
+
+    public void setHandlerFactory(HandlerFactory handlerFactory) {
+        this.handlerFactory = handlerFactory;
+    }
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/Link.java Tue Dec 18 08:47:39 2012
@@ -24,36 +24,35 @@ import org.apache.droids.core.Task;
 
 
 /**
- * Simple extension of a {@link Task}. Adding from/to link, anchor text 
+ * Simple extension of a {@link Task}. Adding from/to link, anchor text
  * and the last modified attribute to the task object.
- * 
+ *
  * @version 1.0
- * 
  */
 public interface Link extends Task {
-  /**
-   * @return the Anchor text for this link
-   */
-  String getAnchorText();
-  
-  /**
-   * From where the link was created
-   * 
-   * @return the parent link from where the link was coming from
-   */
-  Link getFrom();
+    /**
+     * @return the Anchor text for this link
+     */
+    String getAnchorText();
 
-  /**
-   * To where the link is pointing to
-   * 
-   * @return the location where the link is pointing to
-   */
-  Collection<URI> getTo();
+    /**
+     * From where the link was created
+     *
+     * @return the parent link from where the link was coming from
+     */
+    Link getFrom();
 
-  /**
-   * last modified date
-   * 
-   * @return last modified date
-   */
-  Date getLastModifiedDate();
+    /**
+     * To where the link is pointing to
+     *
+     * @return the location where the link is pointing to
+     */
+    Collection<URI> getTo();
+
+    /**
+     * last modified date
+     *
+     * @return last modified date
+     */
+    Date getLastModifiedDate();
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/LinkTask.java Tue Dec 18 08:47:39 2012
@@ -25,171 +25,169 @@ import java.io.Serializable;
 
 
 /**
- * 
  * Basic implementation for @Link. LinkTasks are working instructions for URI
  * based droids.
- * 
  */
 public class LinkTask implements Link, Serializable {
-	private static final long serialVersionUID = -44808094386453088L;
+    private static final long serialVersionUID = -44808094386453088L;
 
-	private Date started;
-	private final int depth;
-	private final URI uri;
-	private final Link from;
-
-	private Date lastModifiedDate;
-	private Collection<URI> linksTo;
-	private String anchorText;
-	private int weight;
-	private boolean aborted = false;
-
-	/**
-	 * Creates a new LinkTask.
-	 * 
-	 * @param from
-	 * @param uri
-	 * @param depth
-	 */
-	public LinkTask(Link from, URI uri, int depth) {
-		this.from = from;
-		this.uri = uri;
-		this.depth = depth;
-		this.started = new Date();
-	}
-
-	/**
-	 * Creates a new LinkTask.
-	 * 
-	 * @param from
-	 * @param uri
-	 * @param depth
-	 * @param weight
-	 */
-	public LinkTask(Link from, URI uri, int depth, int weight) {
-		this.from = from;
-		this.uri = uri;
-		this.depth = depth;
-		this.started = new Date();
-		this.weight = weight;
-	}
-
-	/**
-	 * Creates a new LinkTask
-	 * 
-	 * @param from
-	 * @param uri
-	 * @param depth
-	 * @param anchorText
-	 */
-	public LinkTask(Link from, URI uri, int depth, String anchorText) {
-		this(from, uri, depth);
-		this.anchorText = anchorText;
-	}
-
-	@Override
-	public URI getURI() {
-		return uri;
-	}
-	
-	@Override
-	public InputStream getContent() throws IOException {
-		// TODO Auto-generated method stub
-		return null;
-	}
-
-	@Override
-	public Date getTaskDate() {
-		return started;
-	}
-
-	/**
-	 * Set the Date the task started.
-	 * 
-	 * @param started
-	 */
-	public void setTaskDate(Date started) {
-		this.started = started;
-	}
-
-	@Override
-	public int getDepth() {
-		return depth;
-	}
-
-	@Override
-	public Link getFrom() {
-		return from;
-	}
-
-	@Override
-	public Collection<URI> getTo() {
-		return linksTo;
-	}
-
-	@Override
-	public Date getLastModifiedDate() {
-		return lastModifiedDate;
-	}
-
-	/**
-	 * Set the Date the Task object was last modified.
-	 * 
-	 * @param lastModifiedDate
-	 */
-	public void setLastModifiedDate(Date lastModifiedDate) {
-		this.lastModifiedDate = lastModifiedDate;
-	}
-
-	/**
-	 * Set Outgoing links.
-	 * 
-	 * @param linksTo
-	 */
-	public void setLinksTo(Collection<URI> linksTo) {
-		this.linksTo = linksTo;
-	}
-
-	@Override
-	public String getAnchorText() {
-		return anchorText;
-	}
-
-	/**
-	 * Set the anchor text for this link.
-	 * 
-	 * @param anchorText
-	 */
-	public void setAnchorText(String anchorText) {
-		this.anchorText = anchorText;
-	}
-
-	/**
-	 * Get the weight of the link
-	 * 
-	 * @return the links weight
-	 */
-	public int getWeight() {
-		return weight;
-	}
-
-	/**
-	 * Set the weight of the link.
-	 * 
-	 * @param weight
-	 */
-	public void setWeight(int weight) {
-		this.weight = weight;
-	}
-
-	@Override
-	public void abort() {
-		aborted = true;
-	}
-
-	@Override
-	public boolean isAborted() {
-		return aborted;
-	}
+    private Date started;
+    private final int depth;
+    private final URI uri;
+    private final Link from;
+
+    private Date lastModifiedDate;
+    private Collection<URI> linksTo;
+    private String anchorText;
+    private int weight;
+    private boolean aborted = false;
+
+    /**
+     * Creates a new LinkTask.
+     *
+     * @param from
+     * @param uri
+     * @param depth
+     */
+    public LinkTask(Link from, URI uri, int depth) {
+        this.from = from;
+        this.uri = uri;
+        this.depth = depth;
+        this.started = new Date();
+    }
+
+    /**
+     * Creates a new LinkTask.
+     *
+     * @param from
+     * @param uri
+     * @param depth
+     * @param weight
+     */
+    public LinkTask(Link from, URI uri, int depth, int weight) {
+        this.from = from;
+        this.uri = uri;
+        this.depth = depth;
+        this.started = new Date();
+        this.weight = weight;
+    }
+
+    /**
+     * Creates a new LinkTask
+     *
+     * @param from
+     * @param uri
+     * @param depth
+     * @param anchorText
+     */
+    public LinkTask(Link from, URI uri, int depth, String anchorText) {
+        this(from, uri, depth);
+        this.anchorText = anchorText;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public InputStream getContent() throws IOException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Date getTaskDate() {
+        return started;
+    }
+
+    /**
+     * Set the Date the task started.
+     *
+     * @param started
+     */
+    public void setTaskDate(Date started) {
+        this.started = started;
+    }
+
+    @Override
+    public int getDepth() {
+        return depth;
+    }
+
+    @Override
+    public Link getFrom() {
+        return from;
+    }
+
+    @Override
+    public Collection<URI> getTo() {
+        return linksTo;
+    }
+
+    @Override
+    public Date getLastModifiedDate() {
+        return lastModifiedDate;
+    }
+
+    /**
+     * Set the Date the Task object was last modified.
+     *
+     * @param lastModifiedDate
+     */
+    public void setLastModifiedDate(Date lastModifiedDate) {
+        this.lastModifiedDate = lastModifiedDate;
+    }
+
+    /**
+     * Set Outgoing links.
+     *
+     * @param linksTo
+     */
+    public void setLinksTo(Collection<URI> linksTo) {
+        this.linksTo = linksTo;
+    }
+
+    @Override
+    public String getAnchorText() {
+        return anchorText;
+    }
+
+    /**
+     * Set the anchor text for this link.
+     *
+     * @param anchorText
+     */
+    public void setAnchorText(String anchorText) {
+        this.anchorText = anchorText;
+    }
+
+    /**
+     * Get the weight of the link
+     *
+     * @return the links weight
+     */
+    public int getWeight() {
+        return weight;
+    }
+
+    /**
+     * Set the weight of the link.
+     *
+     * @param weight
+     */
+    public void setWeight(int weight) {
+        this.weight = weight;
+    }
+
+    @Override
+    public void abort() {
+        aborted = true;
+    }
+
+    @Override
+    public boolean isAborted() {
+        return aborted;
+    }
 
 }
\ No newline at end of file

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/crawler/ReportCrawlingDroid.java Tue Dec 18 08:47:39 2012
@@ -19,6 +19,7 @@
 package org.apache.droids.crawler;
 
 import java.util.Queue;
+
 import org.apache.droids.core.TaskMaster;
 import org.apache.droids.core.Worker;
 import org.apache.droids.handle.ReportHandler;
@@ -26,23 +27,21 @@ import org.apache.droids.helper.factorie
 
 
 /**
- * 
  * This simple CrawlingDroid uses the ReportHandler to handle all retrieved files.
- * 
  */
 public class ReportCrawlingDroid extends CrawlingDroid {
 
-  public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
-    super(queue, taskMaster);
-  }
-
-  @Override
-  public Worker<Link> getNewWorker() {
-    final CrawlingWorker worker = new CrawlingWorker( this );
-    HandlerFactory handlerFactory = new HandlerFactory();
-    handlerFactory.getMap().put("default", new ReportHandler());
-    worker.setHandlerFactory(handlerFactory);
-    return worker;
-  }
+    public ReportCrawlingDroid(Queue<Link> queue, TaskMaster<Link> taskMaster) {
+        super(queue, taskMaster);
+    }
+
+    @Override
+    public Worker<Link> getNewWorker() {
+        final CrawlingWorker worker = new CrawlingWorker(this);
+        HandlerFactory handlerFactory = new HandlerFactory();
+        handlerFactory.getMap().put("default", new ReportHandler());
+        worker.setHandlerFactory(handlerFactory);
+        return worker;
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpContentEntity.java Tue Dec 18 08:47:39 2012
@@ -18,51 +18,51 @@
 */
 package org.apache.droids.protocol.http;
 
+import org.apache.droids.core.AdvancedManagedContentEntity;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.droids.core.AdvancedManagedContentEntity;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-
 public class AdvancedHttpContentEntity extends HttpContentEntity implements
-    AdvancedManagedContentEntity {
+        AdvancedManagedContentEntity {
+
+    private Map<String, String> metadata = new HashMap<String, String>();
+    private long contentLength;
+
+    public AdvancedHttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
+        super(entity, maxlen);
+    }
+
+    public AdvancedHttpContentEntity(HttpEntity entity, Header[] allHeaders, long maxlen) throws IOException {
+        super(entity, maxlen);
+        for (Header h : allHeaders) {
+            metadata.put(h.getName(), h.getValue());
+        }
+        contentLength = entity.getContentLength();
+    }
+
+    @Override
+    public boolean containsMetadataKey(String key) {
+        return metadata.containsKey(key);
+    }
 
-  private Map<String,String> metadata = new HashMap<String,String>();
-  private long contentLength;
-  
-  public AdvancedHttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
-    super(entity, maxlen);
-  }
-
-  public AdvancedHttpContentEntity(HttpEntity entity, Header[] allHeaders, long maxlen) throws IOException {
-    super(entity, maxlen);
-    for(Header h : allHeaders) {
-      metadata.put(h.getName(), h.getValue());
-    }
-    contentLength = entity.getContentLength();
-  }
-
-  @Override
-  public boolean containsMetadataKey(String key) {
-    return metadata.containsKey(key);
-  }
-
-  @Override
-  public String getValue(String key) {
-    return metadata.get(key);
-  }
-
-  @Override
-  public Set<String> metadataKeySet() {
-    return metadata.keySet();
-  }
-
-  @Override
-  public long getContentLength() {
-    return this.contentLength;
-  }
+    @Override
+    public String getValue(String key) {
+        return metadata.get(key);
+    }
+
+    @Override
+    public Set<String> metadataKeySet() {
+        return metadata.keySet();
+    }
+
+    @Override
+    public long getContentLength() {
+        return this.contentLength;
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/AdvancedHttpProtocol.java Tue Dec 18 08:47:39 2012
@@ -33,32 +33,32 @@ import org.apache.http.client.methods.Ht
 
 public class AdvancedHttpProtocol extends HttpProtocol {
 
-  public AdvancedHttpProtocol(HttpClient httpclient) {
-    super(httpclient);
-  }
-  
-  public AdvancedHttpProtocol() {
-    super();
-  }
-
-  @Override
-  public AdvancedManagedContentEntity load(URI uri) throws IOException {
-    HttpGet httpget = new HttpGet(uri);
-    HttpResponse response = getHttpClient().execute(httpget);
-    StatusLine statusline = response.getStatusLine();
-    if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
-      httpget.abort();
-      throw new HttpResponseException(
-          statusline.getStatusCode(), statusline.getReasonPhrase());
+    public AdvancedHttpProtocol(HttpClient httpclient) {
+        super(httpclient);
+    }
+
+    public AdvancedHttpProtocol() {
+        super();
     }
-    HttpEntity entity = response.getEntity();
-    if (entity == null) {
-      // Should _almost_ never happen with HTTP GET requests.
-      throw new ClientProtocolException("Empty entity");
+
+    @Override
+    public AdvancedManagedContentEntity load(URI uri) throws IOException {
+        HttpGet httpget = new HttpGet(uri);
+        HttpResponse response = getHttpClient().execute(httpget);
+        StatusLine statusline = response.getStatusLine();
+        if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
+            httpget.abort();
+            throw new HttpResponseException(
+                    statusline.getStatusCode(), statusline.getReasonPhrase());
+        }
+        HttpEntity entity = response.getEntity();
+        if (entity == null) {
+            // Should _almost_ never happen with HTTP GET requests.
+            throw new ClientProtocolException("Empty entity");
+        }
+        long maxlen = getHttpClient().getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
+        return new AdvancedHttpContentEntity(entity, response.getAllHeaders(), maxlen);
     }
-    long maxlen = getHttpClient().getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
-    return new AdvancedHttpContentEntity(entity,response.getAllHeaders(),maxlen);
-  }
 }
 
   

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/ContentTooLongException.java Tue Dec 18 08:47:39 2012
@@ -18,14 +18,12 @@ package org.apache.droids.protocol.http;
 
 import java.io.IOException;
 
-public class ContentTooLongException extends IOException
-{
+public class ContentTooLongException extends IOException {
 
-  private static final long serialVersionUID = -3118026295438863279L;
+    private static final long serialVersionUID = -3118026295438863279L;
+
+    public ContentTooLongException(String message) {
+        super(message);
+    }
 
-  public ContentTooLongException(String message) 
-  {
-    super(message);
-  }
-  
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidHttpEntity.java Tue Dec 18 08:47:39 2012
@@ -27,88 +27,69 @@ import org.apache.http.HttpEntity;
 import org.apache.http.entity.HttpEntityWrapper;
 import org.apache.http.util.ByteArrayBuffer;
 
-class DroidHttpEntity extends HttpEntityWrapper
-{
-  private final byte[] buffer;
-
-  public DroidHttpEntity(final HttpEntity entity, long maxlen) throws IOException
-  {
-    super(entity);
-    if (!entity.isRepeatable() || entity.getContentLength() < 0)
-    {
-      InputStream instream = entity.getContent();
-      ByteArrayBuffer buf = new ByteArrayBuffer(4096);
-      try
-      {
-        byte[] tmp = new byte[4096];
-        long total = 0;
-        int l;
-        while ((l = instream.read(tmp)) != -1)
-        {
-          buf.append(tmp, 0, l);
-          total += l;
-          if (maxlen > 0 && total >= maxlen) {
-            throw new ContentTooLongException("Content length exceeds " + maxlen + " byte limit");
-          }
+class DroidHttpEntity extends HttpEntityWrapper {
+    private final byte[] buffer;
+
+    public DroidHttpEntity(final HttpEntity entity, long maxlen) throws IOException {
+        super(entity);
+        if (!entity.isRepeatable() || entity.getContentLength() < 0) {
+            InputStream instream = entity.getContent();
+            ByteArrayBuffer buf = new ByteArrayBuffer(4096);
+            try {
+                byte[] tmp = new byte[4096];
+                long total = 0;
+                int l;
+                while ((l = instream.read(tmp)) != -1) {
+                    buf.append(tmp, 0, l);
+                    total += l;
+                    if (maxlen > 0 && total >= maxlen) {
+                        throw new ContentTooLongException("Content length exceeds " + maxlen + " byte limit");
+                    }
+                }
+                this.buffer = buf.toByteArray();
+            } finally {
+                instream.close();
+            }
+        } else {
+            this.buffer = null;
         }
-        this.buffer = buf.toByteArray();
-      } finally
-      {
-        instream.close();
-      }
-    } else
-    {
-      this.buffer = null;
-    }
-  }
-
-  public long getContentLength()
-  {
-    if (this.buffer != null)
-    {
-      return this.buffer.length;
-    } else
-    {
-      return wrappedEntity.getContentLength();
-    }
-  }
-
-  public InputStream getContent() throws IOException
-  {
-    if (this.buffer != null)
-    {
-      return new ByteArrayInputStream(this.buffer);
-    } else
-    {
-      return wrappedEntity.getContent();
-    }
-  }
-
-  public boolean isChunked()
-  {
-    return (buffer == null) && wrappedEntity.isChunked();
-  }
-
-  public boolean isRepeatable()
-  {
-    return true;
-  }
-
-  public void writeTo(final OutputStream outstream) throws IOException
-  {
-    Preconditions.checkArgument( outstream != null, "Output stream may not be null" );
-    if (this.buffer != null)
-    {
-      outstream.write(this.buffer);
-    } else
-    {
-      wrappedEntity.writeTo(outstream);
-    }
-  }
-
-  public boolean isStreaming()
-  {
-    return (buffer == null) && wrappedEntity.isStreaming();
-  }
+    }
+
+    public long getContentLength() {
+        if (this.buffer != null) {
+            return this.buffer.length;
+        } else {
+            return wrappedEntity.getContentLength();
+        }
+    }
+
+    public InputStream getContent() throws IOException {
+        if (this.buffer != null) {
+            return new ByteArrayInputStream(this.buffer);
+        } else {
+            return wrappedEntity.getContent();
+        }
+    }
+
+    public boolean isChunked() {
+        return (buffer == null) && wrappedEntity.isChunked();
+    }
+
+    public boolean isRepeatable() {
+        return true;
+    }
+
+    public void writeTo(final OutputStream outstream) throws IOException {
+        Preconditions.checkArgument(outstream != null, "Output stream may not be null");
+        if (this.buffer != null) {
+            outstream.write(this.buffer);
+        } else {
+            wrappedEntity.writeTo(outstream);
+        }
+    }
+
+    public boolean isStreaming() {
+        return (buffer == null) && wrappedEntity.isStreaming();
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpClient.java Tue Dec 18 08:47:39 2012
@@ -70,122 +70,110 @@ import org.apache.http.protocol.RequestU
 /**
  * {@link HttpClient} implementation optimized specifically for web crawling.
  * This HTTP agent has no support for HTTP state management and authentication
- * and is expected to be used for retrieving information from publicly 
- * accessible sites using stateless, idempotent HTTP methods. 
+ * and is expected to be used for retrieving information from publicly
+ * accessible sites using stateless, idempotent HTTP methods.
  */
-public class DroidsHttpClient extends AbstractHttpClient
-{
+public class DroidsHttpClient extends AbstractHttpClient {
 
-  public static final String MAX_BODY_LENGTH = "droids.http..max-body-length";
-  
-  public DroidsHttpClient()
-  {
-    super(null, null);
-  }
-
-  public DroidsHttpClient(HttpParams params)
-  {
-    super(null, params);
-  }
-
-  @Override
-  protected HttpParams createHttpParams()
-  {
-    HttpParams params = new BasicHttpParams();
-    params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
-    params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, HTTP.DEFAULT_CONTENT_CHARSET);
-    params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
-    params.setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
-    params.setIntParameter(CoreConnectionPNames.MAX_HEADER_COUNT, 256);
-    params.setIntParameter(CoreConnectionPNames.MAX_LINE_LENGTH, 5 * 1024);
-    params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 20000);
-    params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000);
-    params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
-    //params.setLongParameter(MAX_BODY_LENGTH, 512 * 1024);
-    return params;
-  }
-
-  @Override
-  protected BasicHttpProcessor createHttpProcessor()
-  {
-    BasicHttpProcessor httpproc = new BasicHttpProcessor();
-    httpproc.addInterceptor(new RequestDefaultHeaders());
-    // Required protocol interceptors
-    httpproc.addInterceptor(new RequestContent());
-    httpproc.addInterceptor(new RequestTargetHost());
-    // Recommended protocol interceptors
-    httpproc.addInterceptor(new RequestConnControl());
-    httpproc.addInterceptor(new RequestUserAgent());
-    httpproc.addInterceptor(new RequestExpectContinue());
-    // HTTP authentication interceptors
-    httpproc.addInterceptor(new RequestProxyAuthentication());
-    return httpproc;
-  }
-
-  @Override
-  protected ClientConnectionManager createClientConnectionManager()
-  {
-    SchemeRegistry schemeRegistry = new SchemeRegistry();
-    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
-    schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
-    return new DroidsHttpConnectionManager(getParams(), schemeRegistry);
-  }
-
-  @Override
-  protected AuthSchemeRegistry createAuthSchemeRegistry()
-  {
-    AuthSchemeRegistry registry = new AuthSchemeRegistry(); 
-    registry.register(
-            AuthPolicy.BASIC, 
-            new BasicSchemeFactory());
-    registry.register(
-            AuthPolicy.DIGEST, 
-            new DigestSchemeFactory());
-    return registry;
-  }
-
-  @Override
-  protected CookieSpecRegistry createCookieSpecRegistry()
-  {
-    // Return empty cookie scheme registry. There'll be no cookie support
-    return new CookieSpecRegistry();
-  }
-
-
-  @Override
-  protected HttpContext createHttpContext()
-  {
-    HttpContext context = new BasicHttpContext();
-    context.setAttribute(
-            ClientContext.AUTHSCHEME_REGISTRY, 
-            getAuthSchemes());
-    context.setAttribute(
-            ClientContext.CREDS_PROVIDER, 
-            getCredentialsProvider());
-    return context;
-  }
-
-  @Override
-  protected HttpRequestRetryHandler createHttpRequestRetryHandler()
-  {
-    return new DroidsRequestRetryHandler();
-  }
-
-  /**
-   * Added ProxySelectorRoutePlanner to support JVM Proxy Settings
-   */
-  @Override
-  protected HttpRoutePlanner createHttpRoutePlanner()
-  {
-    return new ProxySelectorRoutePlanner(
-	        this.getConnectionManager().getSchemeRegistry(),
-	        ProxySelector.getDefault());  
-  }
-
-  @Override
-  protected AuthenticationHandler createTargetAuthenticationHandler()
-  {
-    return new NoAuthHandler();
-  }
+    public static final String MAX_BODY_LENGTH = "droids.http..max-body-length";
+
+    public DroidsHttpClient() {
+        super(null, null);
+    }
+
+    public DroidsHttpClient(HttpParams params) {
+        super(null, params);
+    }
+
+    @Override
+    protected HttpParams createHttpParams() {
+        HttpParams params = new BasicHttpParams();
+        params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
+        params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, HTTP.DEFAULT_CONTENT_CHARSET);
+        params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
+        params.setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
+        params.setIntParameter(CoreConnectionPNames.MAX_HEADER_COUNT, 256);
+        params.setIntParameter(CoreConnectionPNames.MAX_LINE_LENGTH, 5 * 1024);
+        params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 20000);
+        params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000);
+        params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
+        //params.setLongParameter(MAX_BODY_LENGTH, 512 * 1024);
+        return params;
+    }
+
+    @Override
+    protected BasicHttpProcessor createHttpProcessor() {
+        BasicHttpProcessor httpproc = new BasicHttpProcessor();
+        httpproc.addInterceptor(new RequestDefaultHeaders());
+        // Required protocol interceptors
+        httpproc.addInterceptor(new RequestContent());
+        httpproc.addInterceptor(new RequestTargetHost());
+        // Recommended protocol interceptors
+        httpproc.addInterceptor(new RequestConnControl());
+        httpproc.addInterceptor(new RequestUserAgent());
+        httpproc.addInterceptor(new RequestExpectContinue());
+        // HTTP authentication interceptors
+        httpproc.addInterceptor(new RequestProxyAuthentication());
+        return httpproc;
+    }
+
+    @Override
+    protected ClientConnectionManager createClientConnectionManager() {
+        SchemeRegistry schemeRegistry = new SchemeRegistry();
+        schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
+        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
+        return new DroidsHttpConnectionManager(getParams(), schemeRegistry);
+    }
+
+    @Override
+    protected AuthSchemeRegistry createAuthSchemeRegistry() {
+        AuthSchemeRegistry registry = new AuthSchemeRegistry();
+        registry.register(
+                AuthPolicy.BASIC,
+                new BasicSchemeFactory());
+        registry.register(
+                AuthPolicy.DIGEST,
+                new DigestSchemeFactory());
+        return registry;
+    }
+
+    @Override
+    protected CookieSpecRegistry createCookieSpecRegistry() {
+        // Return empty cookie scheme registry. There'll be no cookie support
+        return new CookieSpecRegistry();
+    }
+
+
+    @Override
+    protected HttpContext createHttpContext() {
+        HttpContext context = new BasicHttpContext();
+        context.setAttribute(
+                ClientContext.AUTHSCHEME_REGISTRY,
+                getAuthSchemes());
+        context.setAttribute(
+                ClientContext.CREDS_PROVIDER,
+                getCredentialsProvider());
+        return context;
+    }
+
+    @Override
+    protected HttpRequestRetryHandler createHttpRequestRetryHandler() {
+        return new DroidsRequestRetryHandler();
+    }
+
+    /**
+     * Added ProxySelectorRoutePlanner to support JVM Proxy Settings
+     */
+    @Override
+    protected HttpRoutePlanner createHttpRoutePlanner() {
+        return new ProxySelectorRoutePlanner(
+                this.getConnectionManager().getSchemeRegistry(),
+                ProxySelector.getDefault());
+    }
+
+    @Override
+    protected AuthenticationHandler createTargetAuthenticationHandler() {
+        return new NoAuthHandler();
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnection.java Tue Dec 18 08:47:39 2012
@@ -23,25 +23,22 @@ import org.apache.http.io.HttpMessagePar
 import org.apache.http.io.SessionInputBuffer;
 import org.apache.http.params.HttpParams;
 
-class DroidsHttpConnection extends DefaultClientConnection
-{
+class DroidsHttpConnection extends DefaultClientConnection {
 
-  public DroidsHttpConnection()
-  {
-    super();
-  }
+    public DroidsHttpConnection() {
+        super();
+    }
 
-  @Override
-  protected HttpMessageParser createResponseParser(
-      SessionInputBuffer buffer,
-      HttpResponseFactory responseFactory, 
-      HttpParams params)
-  {
-    return new DefaultResponseParser(
-        buffer, 
-        new LenientHttpResponseParser(), 
-        responseFactory,
-        params);
-  }
+    @Override
+    protected HttpMessageParser createResponseParser(
+            SessionInputBuffer buffer,
+            HttpResponseFactory responseFactory,
+            HttpParams params) {
+        return new DefaultResponseParser(
+                buffer,
+                new LenientHttpResponseParser(),
+                responseFactory,
+                params);
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionManager.java Tue Dec 18 08:47:39 2012
@@ -21,18 +21,15 @@ import org.apache.http.conn.scheme.Schem
 import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
 import org.apache.http.params.HttpParams;
 
-class DroidsHttpConnectionManager extends ThreadSafeClientConnManager
-{
+class DroidsHttpConnectionManager extends ThreadSafeClientConnManager {
 
-  public DroidsHttpConnectionManager(HttpParams params, SchemeRegistry schemes)
-  {
-    super(params, schemes);
-  }
+    public DroidsHttpConnectionManager(HttpParams params, SchemeRegistry schemes) {
+        super(params, schemes);
+    }
+
+    @Override
+    protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schemes) {
+        return new DroidsHttpConnectionOperator(schemes);
+    }
 
-  @Override
-  protected ClientConnectionOperator createConnectionOperator(SchemeRegistry schemes)
-  {
-    return new DroidsHttpConnectionOperator(schemes);
-  }
-  
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsHttpConnectionOperator.java Tue Dec 18 08:47:39 2012
@@ -20,18 +20,15 @@ import org.apache.http.conn.OperatedClie
 import org.apache.http.conn.scheme.SchemeRegistry;
 import org.apache.http.impl.conn.DefaultClientConnectionOperator;
 
-class DroidsHttpConnectionOperator extends DefaultClientConnectionOperator
-{
+class DroidsHttpConnectionOperator extends DefaultClientConnectionOperator {
 
-  public DroidsHttpConnectionOperator(SchemeRegistry schemes)
-  {
-    super(schemes);
-  }
+    public DroidsHttpConnectionOperator(SchemeRegistry schemes) {
+        super(schemes);
+    }
 
-  @Override
-  public OperatedClientConnection createConnection()
-  {
-    return new DroidsHttpConnection();
-  }
+    @Override
+    public OperatedClientConnection createConnection() {
+        return new DroidsHttpConnection();
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/DroidsRequestRetryHandler.java Tue Dec 18 08:47:39 2012
@@ -18,7 +18,7 @@ package org.apache.droids.protocol.http;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
-  import java.net.UnknownHostException;
+import java.net.UnknownHostException;
 
 import javax.net.ssl.SSLHandshakeException;
 
@@ -29,55 +29,51 @@ import org.apache.http.client.HttpReques
 import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.protocol.HttpContext;
 
-class DroidsRequestRetryHandler implements HttpRequestRetryHandler
-{
+class DroidsRequestRetryHandler implements HttpRequestRetryHandler {
 
-  private final int retryCount;
+    private final int retryCount;
 
-  public DroidsRequestRetryHandler(int retryCount)
-  {
-    super();
-    this.retryCount = retryCount;
-  }
-
-  public DroidsRequestRetryHandler()
-  {
-    this(3);
-  }
-
-  @Override
-  public boolean retryRequest(final IOException exception, int executionCount,
-      final HttpContext context)
-  {
-    Preconditions.checkArgument(exception != null, "Exception parameter may not be null" );
-    Preconditions.checkArgument(context != null, "HTTP context may not be null" );
-    
-    if (executionCount > this.retryCount) {
-      // Do not retry if over max retry count
-      return false;
+    public DroidsRequestRetryHandler(int retryCount) {
+        super();
+        this.retryCount = retryCount;
+    }
+
+    public DroidsRequestRetryHandler() {
+        this(3);
+    }
+
+    @Override
+    public boolean retryRequest(final IOException exception, int executionCount,
+                                final HttpContext context) {
+        Preconditions.checkArgument(exception != null, "Exception parameter may not be null");
+        Preconditions.checkArgument(context != null, "HTTP context may not be null");
+
+        if (executionCount > this.retryCount) {
+            // Do not retry if over max retry count
+            return false;
+        }
+        if (exception instanceof NoHttpResponseException) {
+            // Retry if the server dropped connection on us
+            return true;
+        }
+        if (exception instanceof InterruptedIOException) {
+            // Timeout
+            return false;
+        }
+        if (exception instanceof UnknownHostException) {
+            // Unknown host
+            return false;
+        }
+        if (exception instanceof HttpHostConnectException) {
+            // Connection refused
+            return false;
+        }
+        if (exception instanceof SSLHandshakeException) {
+            // SSL handshake exception
+            return false;
+        }
+        // otherwise retry
+        return true;
     }
-    if (exception instanceof NoHttpResponseException) {
-      // Retry if the server dropped connection on us
-      return true;
-    }
-    if (exception instanceof InterruptedIOException) {
-      // Timeout
-      return false;
-    }
-    if (exception instanceof UnknownHostException) {
-      // Unknown host
-      return false;
-    }
-    if (exception instanceof HttpHostConnectException) {
-      // Connection refused
-      return false;
-    }
-    if (exception instanceof SSLHandshakeException) {
-      // SSL handshake exception
-      return false;
-    }
-    // otherwise retry
-    return true;
-  }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpClientContentLoader.java Tue Dec 18 08:47:39 2012
@@ -36,40 +36,38 @@ import org.apache.http.client.methods.Ht
 /**
  * {@link ContentLoader} based on HttpClient 4.0.
  */
-public class HttpClientContentLoader implements ContentLoader
-{
+public class HttpClientContentLoader implements ContentLoader {
 
-  private final HttpClient httpclient;
-  
-  public HttpClientContentLoader(HttpClient httpclient) {
-    super();
-    this.httpclient = httpclient;
-  }
-  
-  public boolean exists(URI uri) throws IOException
-  {
-    HttpHead httphead = new HttpHead(uri);
-    HttpResponse response = httpclient.execute(httphead);
-    return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
-  }
+    private final HttpClient httpclient;
 
-  public InputStream load(URI uri) throws IOException {
-    HttpGet httpget = new HttpGet(uri);
-    HttpResponse response = httpclient.execute(httpget);
-    StatusLine statusline = response.getStatusLine();
-    if (statusline.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
-      return null;
+    public HttpClientContentLoader(HttpClient httpclient) {
+        super();
+        this.httpclient = httpclient;
     }
-    if (statusline.getStatusCode() != HttpStatus.SC_OK) {
-      throw new HttpResponseException(
-          statusline.getStatusCode(), statusline.getReasonPhrase());
+
+    public boolean exists(URI uri) throws IOException {
+        HttpHead httphead = new HttpHead(uri);
+        HttpResponse response = httpclient.execute(httphead);
+        return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
     }
-    HttpEntity entity = response.getEntity();
-    if (entity != null) {
-      return entity.getContent();
-    } else {
-      return null;
+
+    public InputStream load(URI uri) throws IOException {
+        HttpGet httpget = new HttpGet(uri);
+        HttpResponse response = httpclient.execute(httpget);
+        StatusLine statusline = response.getStatusLine();
+        if (statusline.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
+            return null;
+        }
+        if (statusline.getStatusCode() != HttpStatus.SC_OK) {
+            throw new HttpResponseException(
+                    statusline.getStatusCode(), statusline.getReasonPhrase());
+        }
+        HttpEntity entity = response.getEntity();
+        if (entity != null) {
+            return entity.getContent();
+        } else {
+            return null;
+        }
     }
-  }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpContentEntity.java Tue Dec 18 08:47:39 2012
@@ -30,77 +30,77 @@ import org.apache.http.protocol.HTTP;
 
 public class HttpContentEntity implements ManagedContentEntity {
 
-  private final HttpEntity entity;
-  private final String mimeType;
-  private final String charset;
-  
-  private Parse parse = null;
-  
-  public HttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
-    super();
-    if (entity.isRepeatable()) {
-      this.entity = entity;
-    } else {
-      this.entity = new DroidHttpEntity(entity, maxlen);
-    }
-    
-    String mimeType = null;
-    String charset = null;
-    Header header = entity.getContentType();
-    if (header != null) {
-      HeaderElement[] helems = header.getElements();
-      if (helems != null && helems.length > 0) {
-        HeaderElement helem = helems[0];
-        mimeType = helem.getName();
-        NameValuePair nvp = helem.getParameterByName("charset");
-        if (nvp != null) {
-          charset = nvp.getValue();
+    private final HttpEntity entity;
+    private final String mimeType;
+    private final String charset;
+
+    private Parse parse = null;
+
+    public HttpContentEntity(HttpEntity entity, long maxlen) throws IOException {
+        super();
+        if (entity.isRepeatable()) {
+            this.entity = entity;
+        } else {
+            this.entity = new DroidHttpEntity(entity, maxlen);
         }
-      }
+
+        String mimeType = null;
+        String charset = null;
+        Header header = entity.getContentType();
+        if (header != null) {
+            HeaderElement[] helems = header.getElements();
+            if (helems != null && helems.length > 0) {
+                HeaderElement helem = helems[0];
+                mimeType = helem.getName();
+                NameValuePair nvp = helem.getParameterByName("charset");
+                if (nvp != null) {
+                    charset = nvp.getValue();
+                }
+            }
+        }
+        if (mimeType != null) {
+            this.mimeType = mimeType.toLowerCase(Locale.ENGLISH);
+        } else {
+            this.mimeType = "binary/octet-stream";
+        }
+        if (charset != null) {
+            this.charset = charset;
+        } else {
+            if (this.mimeType.startsWith("text/")) {
+                this.charset = HTTP.ISO_8859_1;
+            } else {
+                this.charset = null;
+            }
+        }
+    }
+
+    @Override
+    public String getMimeType() {
+        return mimeType;
+    }
+
+    @Override
+    public String getCharset() {
+        return charset;
+    }
+
+    @Override
+    public InputStream obtainContent() throws IOException {
+        return entity.getContent();
+    }
+
+    @Override
+    public Parse getParse() {
+        return this.parse;
+    }
+
+    @Override
+    public void setParse(Parse parse) {
+        this.parse = parse;
+    }
+
+    @Override
+    public void finish() {
     }
-    if (mimeType != null) {
-      this.mimeType = mimeType.toLowerCase(Locale.ENGLISH);
-    } else {
-      this.mimeType = "binary/octet-stream";
-    }
-    if (charset != null) {
-      this.charset = charset;
-    } else {
-      if (this.mimeType.startsWith("text/")) {
-        this.charset = HTTP.ISO_8859_1;
-      } else {
-        this.charset = null;
-      }
-    }
-  }
-
-  @Override
-  public String getMimeType() {
-    return mimeType;
-  }
-
-  @Override
-  public String getCharset() {
-    return charset;
-  }
-
-  @Override
-  public InputStream obtainContent() throws IOException {
-    return entity.getContent();
-  }
-
-  @Override
-  public Parse getParse() {
-    return this.parse;
-  }
-
-  @Override
-  public void setParse(Parse parse) {
-    this.parse = parse;
-  }
-
-  @Override
-  public void finish() {
-  }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/HttpProtocol.java Tue Dec 18 08:47:39 2012
@@ -42,111 +42,110 @@ import org.slf4j.LoggerFactory;
  */
 public class HttpProtocol implements Protocol {
 
-  private static final Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+    private static final Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
 
-  private final HttpClient httpclient;
-  private final ContentLoader contentLoader;
-  
-  private boolean forceAllow = false;
-  private String userAgent = "Apache-Droids/1.1 (java 1.5)";
-
-  public HttpProtocol(final HttpClient httpclient) {
-    super();
-    this.httpclient = httpclient;
-    this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
-    this.contentLoader = new HttpClientContentLoader(httpclient);
-  }
-  
-  public HttpProtocol() {
-    this(new DroidsHttpClient());
-  }
-
-  @Override
-  public ManagedContentEntity load(URI uri) throws IOException {
-    HttpGet httpget = new HttpGet(uri);
-    HttpResponse response = httpclient.execute(httpget);
-    StatusLine statusline = response.getStatusLine();
-    if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
-      httpget.abort();
-      throw new HttpResponseException(
-          statusline.getStatusCode(), statusline.getReasonPhrase());
-    }
-    HttpEntity entity = response.getEntity();
-    if (entity == null) {
-      // Should _almost_ never happen with HTTP GET requests.
-      throw new ClientProtocolException("Empty entity");
-    }
-    long maxlen = httpclient.getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
-    return new HttpContentEntity(entity, maxlen);
-  }
-
-  @Override
-  public boolean isAllowed(URI uri) throws IOException {
-    if (forceAllow) {
-      return forceAllow;
-    }
-
-    URI baseURI;
-    try {
-      baseURI = new URI(
-          uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), 
-          "/", null, null);
-    } catch (URISyntaxException ex) {
-      LOG.error("Unable to determine base URI for " + uri);
-      return false;
-    }
-    
-    NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
-    try {
-      nrc.parse(baseURI);
-    } catch (NoRobotException ex) {
-      LOG.error("Failure parsing robots.txt: " + ex.getMessage());
-      return false;
-    }
-    boolean test = nrc.isUrlAllowed(uri);
-    if (LOG.isInfoEnabled()) {
-      LOG.info(uri + " is " + (test ? "allowed" : "denied"));
-    }
-    return test;
-  }
-
-  public String getUserAgent() {
-    return userAgent;
-  }
-
-  public void setUserAgent(String userAgent) {
-    this.userAgent = userAgent;
-    this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
-  }
-
-  /**
-   * You can force that a site is allowed (ignoring the robots.txt). This should
-   * only be used on server that you control and where you have the permission
-   * to ignore the robots.txt.
-   * 
-   * @return <code>true</code> if you are rude and ignore robots.txt.
-   *         <code>false</code> if you are playing nice.
-   */
-  public boolean isForceAllow() {
-    return forceAllow;
-  }
-
-  /**
-   * You can force that a site is allowed (ignoring the robot.txt). This should
-   * only be used on server that you control and where you have the permission
-   * to ignore the robots.txt.
-   * 
-   * @param forceAllow
-   *                if you want to force an allow and ignore the robot.txt set
-   *                to <code>true</code>. If you want to obey the rules and
-   *                be polite set to <code>false</code>.
-   */
-  public void setForceAllow(boolean forceAllow) {
-    this.forceAllow = forceAllow;
-  }
-  
-  protected HttpClient getHttpClient() {
-    return this.httpclient;
-  }
+    private final HttpClient httpclient;
+    private final ContentLoader contentLoader;
+
+    private boolean forceAllow = false;
+    private String userAgent = "Apache-Droids/1.1 (java 1.5)";
+
+    public HttpProtocol(final HttpClient httpclient) {
+        super();
+        this.httpclient = httpclient;
+        this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
+        this.contentLoader = new HttpClientContentLoader(httpclient);
+    }
+
+    public HttpProtocol() {
+        this(new DroidsHttpClient());
+    }
+
+    @Override
+    public ManagedContentEntity load(URI uri) throws IOException {
+        HttpGet httpget = new HttpGet(uri);
+        HttpResponse response = httpclient.execute(httpget);
+        StatusLine statusline = response.getStatusLine();
+        if (statusline.getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
+            httpget.abort();
+            throw new HttpResponseException(
+                    statusline.getStatusCode(), statusline.getReasonPhrase());
+        }
+        HttpEntity entity = response.getEntity();
+        if (entity == null) {
+            // Should _almost_ never happen with HTTP GET requests.
+            throw new ClientProtocolException("Empty entity");
+        }
+        long maxlen = httpclient.getParams().getLongParameter(DroidsHttpClient.MAX_BODY_LENGTH, 0);
+        return new HttpContentEntity(entity, maxlen);
+    }
+
+    @Override
+    public boolean isAllowed(URI uri) throws IOException {
+        if (forceAllow) {
+            return forceAllow;
+        }
+
+        URI baseURI;
+        try {
+            baseURI = new URI(
+                    uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(),
+                    "/", null, null);
+        } catch (URISyntaxException ex) {
+            LOG.error("Unable to determine base URI for " + uri);
+            return false;
+        }
+
+        NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
+        try {
+            nrc.parse(baseURI);
+        } catch (NoRobotException ex) {
+            LOG.error("Failure parsing robots.txt: " + ex.getMessage());
+            return false;
+        }
+        boolean test = nrc.isUrlAllowed(uri);
+        if (LOG.isInfoEnabled()) {
+            LOG.info(uri + " is " + (test ? "allowed" : "denied"));
+        }
+        return test;
+    }
+
+    public String getUserAgent() {
+        return userAgent;
+    }
+
+    public void setUserAgent(String userAgent) {
+        this.userAgent = userAgent;
+        this.httpclient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgent);
+    }
+
+    /**
+     * You can force that a site is allowed (ignoring the robots.txt). This should
+     * only be used on server that you control and where you have the permission
+     * to ignore the robots.txt.
+     *
+     * @return <code>true</code> if you are rude and ignore robots.txt.
+     *         <code>false</code> if you are playing nice.
+     */
+    public boolean isForceAllow() {
+        return forceAllow;
+    }
+
+    /**
+     * You can force that a site is allowed (ignoring the robot.txt). This should
+     * only be used on server that you control and where you have the permission
+     * to ignore the robots.txt.
+     *
+     * @param forceAllow if you want to force an allow and ignore the robot.txt set
+     *                   to <code>true</code>. If you want to obey the rules and
+     *                   be polite set to <code>false</code>.
+     */
+    public void setForceAllow(boolean forceAllow) {
+        this.forceAllow = forceAllow;
+    }
+
+    protected HttpClient getHttpClient() {
+        return this.httpclient;
+    }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/LenientHttpResponseParser.java Tue Dec 18 08:47:39 2012
@@ -22,20 +22,16 @@ import org.apache.http.message.BasicHead
 import org.apache.http.message.BasicLineParser;
 import org.apache.http.util.CharArrayBuffer;
 
-class LenientHttpResponseParser extends BasicLineParser
-{
+class LenientHttpResponseParser extends BasicLineParser {
 
-  @Override
-  public Header parseHeader(CharArrayBuffer buffer) throws ParseException
-  {
-    try
-    {
-      return super.parseHeader(buffer);
-    } catch (ParseException ex)
-    {
-      // Suppress ParseException exception
-      return new BasicHeader(buffer.toString(), null);
+    @Override
+    public Header parseHeader(CharArrayBuffer buffer) throws ParseException {
+        try {
+            return super.parseHeader(buffer);
+        } catch (ParseException ex) {
+            // Suppress ParseException exception
+            return new BasicHeader(buffer.toString(), null);
+        }
     }
-  }
 
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/main/java/org/apache/droids/protocol/http/NoAuthHandler.java Tue Dec 18 08:47:39 2012
@@ -27,28 +27,24 @@ import org.apache.http.auth.MalformedCha
 import org.apache.http.client.AuthenticationHandler;
 import org.apache.http.protocol.HttpContext;
 
-class NoAuthHandler implements AuthenticationHandler
-{
+class NoAuthHandler implements AuthenticationHandler {
 
-  @Override
-  public Map<String, Header> getChallenges(HttpResponse response, HttpContext context)
-      throws MalformedChallengeException
-  {
-    return Collections.emptyMap();
-  }
+    @Override
+    public Map<String, Header> getChallenges(HttpResponse response, HttpContext context)
+            throws MalformedChallengeException {
+        return Collections.emptyMap();
+    }
 
-  @Override
-  public boolean isAuthenticationRequested(HttpResponse response, HttpContext context)
-  {
-    return false;
-  }
+    @Override
+    public boolean isAuthenticationRequested(HttpResponse response, HttpContext context) {
+        return false;
+    }
+
+    @Override
+    public AuthScheme selectScheme(Map<String, Header> challenges, HttpResponse response, HttpContext context)
+            throws AuthenticationException {
+        throw new AuthenticationException(
+                "Unable to respond to any of these challenges: " + challenges);
+    }
 
-  @Override
-  public AuthScheme selectScheme(Map<String, Header> challenges, HttpResponse response, HttpContext context)
-      throws AuthenticationException
-  {
-    throw new AuthenticationException(
-        "Unable to respond to any of these challenges: " + challenges);
-  }
-  
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-crawler/src/test/java/org/apache/droids/AppTest.java Tue Dec 18 08:47:39 2012
@@ -7,32 +7,28 @@ import junit.framework.TestSuite;
 /**
  * Unit test for simple App.
  */
-public class AppTest 
-    extends TestCase
-{
+public class AppTest
+        extends TestCase {
     /**
      * Create the test case
      *
      * @param testName name of the test case
      */
-    public AppTest( String testName )
-    {
-        super( testName );
+    public AppTest(String testName) {
+        super(testName);
     }
 
     /**
      * @return the suite of tests being tested
      */
-    public static Test suite()
-    {
-        return new TestSuite( AppTest.class );
+    public static Test suite() {
+        return new TestSuite(AppTest.class);
     }
 
     /**
      * Rigourous Test :-)
      */
-    public void testApp()
-    {
-        assertTrue( true );
+    public void testApp() {
+        assertTrue(true);
     }
 }

Modified: incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml
URL: http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml?rev=1423339&r1=1423338&r2=1423339&view=diff
==============================================================================
--- incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml (original)
+++ incubator/droids/branches/0.2.x-cleanup/droids-examples/pom.xml Tue Dec 18 08:47:39 2012
@@ -25,56 +25,57 @@
    <http://www.apache.org />.
  -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <artifactId>droids</artifactId>
-    <groupId>org.apache.droids</groupId>
-    <version>0.3.0-incubating-SNAPSHOT</version>
-  </parent>
-  <artifactId>droids-examples</artifactId>
-  <name>Apache Droids Examples</name>
-  <inceptionYear>2007</inceptionYear>
-  <description>
-   Apache Droids Examples
-  </description>
-  <packaging>jar</packaging>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>droids</artifactId>
+        <groupId>org.apache.droids</groupId>
+        <version>0.3.0-incubating-SNAPSHOT</version>
+    </parent>
+    <artifactId>droids-examples</artifactId>
+    <name>Apache Droids Examples</name>
+    <inceptionYear>2007</inceptionYear>
+    <description>
+        Apache Droids Examples
+    </description>
+    <packaging>jar</packaging>
 
 
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.droids</groupId>
-      <artifactId>droids-core</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.droids</groupId>
-      <artifactId>droids-tika</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>${junit.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <version>1.8.5</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.droids</groupId>
-      <artifactId>droids-core</artifactId>
-      <version>${project.version}</version>
-      <type>test-jar</type>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-simple</artifactId>
-      <version>1.6.4</version>
-    </dependency>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.droids</groupId>
+            <artifactId>droids-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.droids</groupId>
+            <artifactId>droids-tika</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <version>1.8.5</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.droids</groupId>
+            <artifactId>droids-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>1.6.4</version>
+        </dependency>
 
-  </dependencies>
+    </dependencies>
 </project>



Mime
View raw message