incubator-droids-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ming...@apache.org
Subject svn commit: r810273 [4/4] - in /incubator/droids/trunk/droids-crawler: ./ docs/ docs/diagrams/ src/ src/main/ src/main/groovy/ src/main/java/ src/main/java/org/ src/main/java/org/apache/ src/main/java/org/apache/droids/ src/main/java/org/apache/droids/...
Date Tue, 01 Sep 2009 22:11:33 GMT
Added: incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java (added)
+++ incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/Weighted.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.droids.crawler.util;
+
+/**
+ * For the filter framework, a filter with a higher weight (i.e. more important) shall be executed first.
+ */
+public interface Weighted {
+    int getWeight();
+}

Added: incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java (added)
+++ incubator/droids/trunk/droids-crawler/src/main/java/org/apache/droids/crawler/util/logging/LogUtils.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.util.logging;
+
+import java.util.Map;
+import java.util.Collection;
+
+public class LogUtils{
+
+    /**
+     * convert any object to String in a way similar to Groovy syntax.
+     * e.g. any string -> "any string"
+     * map of key and value -> [key:value]
+     * collection of v1, v2 -> [v1, v2]
+     */
+    public static String toString(Object target){
+        return append(target, new StringBuilder()).toString();
+    }
+
+    /**
+     * This method creates a String representation of any target in a way similar to Groovy syntax
+     */
+    public static StringBuilder append(Object target, StringBuilder out){
+        if (target instanceof String){
+            out.append("\"").append(target).append("\"");
+        } else if (target instanceof Boolean || target instanceof Integer){
+            out.append(target);
+        } else if (target instanceof Map){
+            Map map = (Map) target;
+            boolean first = true;
+            out.append("[");
+            if (map.size() == 0){
+                out.append(":");
+            } else{
+                for (Object key : map.keySet()){
+                    if (!first) out.append(", ");
+                    append(key, out);
+                    out.append(":");
+                    append(map.get(key), out);
+                    first = false;
+                }
+            }
+            out.append("]");
+        } else if (target instanceof Collection){
+            boolean first = true;
+            out.append("[");
+            for (Object t : (Collection) target){
+                if (!first) out.append(",");
+                append(t, out);
+                first = false;
+            }
+            out.append("]");
+        } else{
+            out.append(target != null ? target.toString() : "null");
+        }
+        return out;
+    }
+
+}

Added: incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/main/resources/log4j.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration debug="false" xmlns:log4j="http://jakarta.apache.org/log4j/">
+    <!-- http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html -->
+
+    <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p [%-40.40c] [%-11t]  - %m%n"/>
+            <!--<param name="ConversionPattern" value="%d %-5p [%-30.40c{1}] [%t]  - %m%n"/>-->
+        </layout>
+    </appender>
+
+    <logger name="org.apache.http.wire" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.apache.droids.crawler" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    
+    <root>
+        <priority value="info"/>
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+</log4j:configuration>

Added: incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/main/resources/sample-crawler-service-cloud.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+
+    <!-- for multiple remote crawler service -->
+    <bean id="crawlerService" class="org.apache.droids.crawler.RoundRobinCrawlerService">
+        <property name="crawlerServices">
+            <util:list>
+                <bean id="remoteCrawlerService0"
+                      class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+                    <!-- TODO: change the following serviceUrl -->
+                    <property name="serviceUrl" value="http://YOUR_APP0.appspot.com"/>
+                    <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+                </bean>
+                <bean id="remoteCrawlerService1"
+                      class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+                    <!-- TODO: change the following serviceUrl -->
+                    <property name="serviceUrl" value="http://YOUR_APP1.appspot.com"/>
+                    <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+                </bean>
+            </util:list>
+        </property>
+    </bean>
+
+    <!-- for single remote crawler service -->
+    <!--
+    <bean id="crawlerService" class="org.springframework.remoting.httpinvoker.HttpInvokerProxyFactoryBean">
+        <property name="serviceUrl" value="http://YOUR_APP0.appspot.com"/>
+        <property name="serviceInterface" value="org.apache.droids.crawler.CrawlerService"/>
+    </bean>
+    -->
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DependencyTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertNotNull;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+public class DependencyTest{
+    @Test public void testSpring(){
+        assertNotNull(new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml"));
+        //assertNotNull(new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml").getBean("httpClient"));
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/DummyCrawlerService.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.extractor.ExtractorException;
+
+/**
+ * This is for unit testing only
+ */
+public class DummyCrawlerService implements CrawlerService<Link>{
+    protected int fetchCount, parseCount, extractCount;
+
+    public String getVersion(){
+        return null;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
+    public String getNode(){
+        return null;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
+    public Link fetch(Link link) throws FetcherException{
+        fetchCount++;
+        return link;
+    }
+
+    public Link parse(Link link) throws ParserException{
+        parseCount++;
+        return link;
+    }
+
+    public Link extract(Link link) throws ExtractorException{
+        extractCount++;
+        return link;
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LinkTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+
+public class LinkTest{
+
+    @Test public void testType(){
+        assertTrue(new Link() instanceof Map);
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/LocalCrawlerServiceTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.fetcher.Fetcher;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.extractor.ExtractorException;
+
+import java.net.URISyntaxException;
+import java.util.Set;
+import java.util.Map;
+
+public class LocalCrawlerServiceTest{
+    ApplicationContext context = new ClassPathXmlApplicationContext("test-LocalCrawlerServiceTest.xml");
+    static String url = "http://www.apache.org";
+
+    @Test public void testFetch() throws URISyntaxException, FetcherException{
+        CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+        assertNotNull(crawlerService);
+        Link fetchedLink = crawlerService.fetch(new Link(url));
+        assertNotNull(fetchedLink);
+        assertNotNull(fetchedLink.get("fetched"));
+        assertTrue(fetchedLink.get("fetched") instanceof Fetcher);
+        assertNotNull(fetchedLink.get("fetched", Fetcher.class).getEntity());
+    }
+
+    @Test public void testFetchParse() throws URISyntaxException, FetcherException, ParserException{
+        CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+        Link fetchedLink = crawlerService.fetch(new Link(url));
+        Link parsedLink = crawlerService.parse(fetchedLink);
+        assertTrue(parsedLink.get("parsed") instanceof Parser);
+        assertNotNull(parsedLink.get("parsed", Parser.class).getData());
+    }
+
+    @Test public void testParseWithoutFetch() throws URISyntaxException, FetcherException, ParserException{
+        CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+        Link parsedLink = crawlerService.parse(new Link(url));
+        assertTrue(parsedLink.get("parsed") instanceof Parser);
+        Map<String, Map<String, Set<String>>> data = (Map<String, Map<String, Set<String>>>) parsedLink.get("parsed", Parser.class).getData();
+        assertNotNull(data);
+        assertEquals(3, data.size());
+    }
+
+    @Test public void testExtractWithoutFetchAndParse() throws ExtractorException, URISyntaxException{
+        CrawlerService crawlerService = context.getBean("crawlerService", CrawlerService.class);
+        Link link = crawlerService.extract(new Link(url));
+        assertFalse(link.containsKey("parsed"));
+        assertTrue(link.containsKey("extracted"));
+        assertTrue(link.get("extracted") instanceof Set);
+        Set<Link> outlinks = link.get("extracted", Set.class);
+        System.out.println(outlinks);
+        assertTrue(outlinks.size() > 100);
+    }
+
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/RoundRobinCrawlerServiceTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.context.ApplicationContext;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.apache.droids.crawler.fetcher.FetcherException;
+
+import java.net.URISyntaxException;
+
+public class RoundRobinCrawlerServiceTest{
+    ApplicationContext context = new ClassPathXmlApplicationContext("test-RoundRobinCrawlerServiceTest.xml");
+
+    @Test public void testFetch() throws FetcherException, URISyntaxException{
+        RoundRobinCrawlerService service = context.getBean("crawlerService", RoundRobinCrawlerService.class);
+        assertEquals(3, service.getCrawlerServices().size());
+        for (int i = 0; i < 30; i++){
+            service.fetch(new Link("http://www.apache.org/page/" + i));
+        }
+
+        for (Object delegate : service.getCrawlerServices()){
+            DummyCrawlerService dummyDelegate = (DummyCrawlerService) delegate;
+            assertEquals(10, dummyDelegate.fetchCount);
+        }
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/SimpleCrawlerControllerTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler;
+
+import org.junit.Test;
+import static org.junit.Assert.assertNotNull;
+import org.apache.droids.exception.DroidsException;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.context.ApplicationContext;
+
+public class SimpleCrawlerControllerTest{
+    ApplicationContext context = new ClassPathXmlApplicationContext("test-StandaloneCrawlerControllerTest.xml");
+
+    @Test public void testCrawlerControllerInit() throws DroidsException{
+        StandaloneCrawlerController controller = new StandaloneCrawlerController();
+        controller.init();
+        assertNotNull(controller.getQueue());
+    }
+
+    @Test public void testCrawlerControllerStart() throws CrawlerException{
+        CrawlerController controller = context.getBean("crawlerController", CrawlerController.class);
+        controller.start();
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/extractor/HtmlElementLinkExtractorTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.extractor;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.filter.extract.IncludeFilter;
+import org.apache.droids.crawler.filter.extract.RefererFilter;
+import org.apache.droids.crawler.util.WeightComparator;
+import org.apache.droids.crawler.fetcher.HttpFetcher;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.parser.ParserException;
+import org.apache.droids.crawler.parser.impl.NekoHtmlParser;
+import org.apache.droids.crawler.parser.impl.AbstractHierarchicalDataParser;
+import org.apache.droids.core.Entity;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.regex.Matcher;
+import java.net.URISyntaxException;
+
+public class HtmlElementLinkExtractorTest{
+    ApplicationContext context = new ClassPathXmlApplicationContext("test-extractor.HtmlElementLinkExtractorTest.xml");
+
+    @Test public void testParser() throws URISyntaxException, ParserException, FetcherException{
+        Link link = new Link("http://www.apache.org");
+        Parser parser = context.getBean("nekoHtmlParser0", NekoHtmlParser.class);
+        parser.parse(link, new HttpFetcher().fetch(link).getEntity(), null);
+        Set links = context.getBean("htmlElementLinkExtractor", HtmlElementLinkExtractor.class).extract(link, parser);
+        assertNotNull(links);
+        assertTrue(links.size() > 100);
+    }
+
+
+    @Test public void testExtractStyleBackground() throws URISyntaxException{
+        Link link = new Link("http://www.w3schools.com/css/tryit_view.asp?filename=trycss_background-image");
+        final Map data = new HashMap(), style = new HashMap();
+        Set blank = new HashSet();
+        data.put("STYLE", style);
+        style.put("", blank);
+        blank.add("body {background-image:url('paper.gif')}");
+
+        Set<Link> links = context.getBean("htmlElementLinkExtractor", Extractor.class).extract(link, new AbstractHierarchicalDataParser<Link>(){
+            public Parser parse(Link link, Entity entity, Map<String, Object> params) throws ParserException{return null; }
+
+            @Override public Map getData(){return data;}
+
+            public void reset(){}
+
+            public boolean matches(Link link){return false;}
+        });
+        assertEquals(1, links.size());
+        assertEquals("http://www.w3schools.com/css/paper.gif", links.iterator().next().getUrl());
+    }
+
+    @Test public void testExtractFilterSorting(){ //TODO move to another test class
+
+        IncludeFilter includeFilter = new IncludeFilter("");
+        includeFilter.setWeight(100);
+        RefererFilter refererFilter = new RefererFilter();
+        refererFilter.setWeight(200);
+
+        HtmlElementLinkExtractor extractor = new HtmlElementLinkExtractor();
+        extractor.addFilter(includeFilter);
+        extractor.addFilter(refererFilter);
+        extractor.setFilterComparator(new WeightComparator());
+        extractor.init();
+
+        assertTrue(extractor.filters.get(0) instanceof RefererFilter);
+
+    }
+
+    @Test public void testCSSBackground(){
+        Matcher matcher = null;
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url('paper.gif')}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url(\"paper.gif\")}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url(paper.gif)}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image :url(paper.gif)}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image: url(paper.gif)}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url( 'paper.gif' )}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+
+        matcher = HtmlElementLinkExtractor.CSS_BG.matcher("body {background-image:url( paper.gif )}");
+        assertTrue(matcher.matches());
+        assertEquals("paper.gif", matcher.group(1));
+    }
+
+
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/HttpFetcherTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.fetcher;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import org.apache.droids.crawler.Link;
+import org.apache.commons.io.IOUtils;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.net.URISyntaxException;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+//TODO find a better way to test than making http request to apache
+public class HttpFetcherTest{
+
+    @Test public void testDefaultFetch() throws FetcherException, URISyntaxException, IOException{
+        Fetcher fetcher = new HttpFetcher().fetch(new Link("http://incubator.apache.org/droids/"));
+        assertEquals(200, fetcher.getStatusCode());
+        assertNotNull(fetcher.getEntity());
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copy(fetcher.getEntity().getContent(), out);
+        assertTrue(out.toByteArray().length > 15000);
+    }
+
+    @Test public void testDefaultFetch_Spring() throws FetcherException, URISyntaxException{
+        Fetcher fetcher = new ClassPathXmlApplicationContext("test-fetcher.HttpFetcherTest.xml").getBean("httpFetcher", HttpFetcher.class);
+        fetcher.fetch(new Link("http://incubator.apache.org/droids/"), null);
+        assertEquals(200, fetcher.getStatusCode());
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/fetcher/appengine/AppEngineFetcherTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.fetcher.appengine;
+
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import org.apache.droids.crawler.Link;
+
+import java.net.URISyntaxException;
+
+public class AppEngineFetcherTest{
+
+    @Test public void testMatch() throws URISyntaxException{
+        assertTrue(new AppEngineFetcher().matches(new Link("http://www.apache.org")));
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/filter/StatsFilterTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.filter;
+
+import org.apache.droids.crawler.Link;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+
+public class StatsFilterTest{
+
+    @Test public void testPollCount() throws URISyntaxException{
+        StatsFilter filter = new StatsFilter().init();
+
+        for (int i = 0; i < 10; i++) filter.polled(new Link("http://www.apache.org/page/" + i));
+
+        Map report = filter.report();
+        assertEquals(9, report.size());
+        assertEquals(10, report.get("poll.count"));
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/DefaultParserFactoryTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,34 @@
+package org.apache.droids.crawler.parser;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.parser.impl.NekoHtmlParser;
+import static junit.framework.Assert.assertEquals;
+
+import java.net.URISyntaxException;
+
+public class DefaultParserFactoryTest{
+
+    @Test public void testAutowiringOfParsers(){
+        ApplicationContext context = new ClassPathXmlApplicationContext("test-parser.DefaultParserFactoryTest.xml");
+        DefaultParserFactory factory = context.getBean("parserFactory", DefaultParserFactory.class);
+        assertNotNull(factory);
+        assertNotNull(factory.getParsers());
+        assertEquals(3, factory.getParsers().size());
+    }
+
+    @Test public void testTextHtml() throws URISyntaxException{
+        ApplicationContext context = new ClassPathXmlApplicationContext("test-parser.DefaultParserFactoryTest.xml");
+        DefaultParserFactory factory = context.getBean("parserFactory", DefaultParserFactory.class);
+        Link link = new Link("http://www.apache.org");
+        link.put("contentType", "text/html");
+        Parser parser = factory.newParser(link);
+        assertNotNull(parser);
+        assertTrue(parser instanceof NekoHtmlParser);
+    }
+
+}
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/parser/impl/NekoHtmlParserTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,52 @@
+package org.apache.droids.crawler.parser.impl;
+
+import org.apache.droids.core.Entity;
+import org.apache.droids.crawler.Link;
+import org.apache.droids.crawler.fetcher.FetcherException;
+import org.apache.droids.crawler.fetcher.HttpFetcher;
+import org.apache.droids.crawler.parser.Parser;
+import org.apache.droids.crawler.parser.ParserException;
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Set;
+
+
+public class NekoHtmlParserTest{
+    /*ApplicationContext context;
+
+    @Before public void setUp(){
+        context = ;
+    }*/
+
+    @Test public void testParser_A_href() throws URISyntaxException, FetcherException, ParserException{
+        Link link = new Link("http://www.apache.org/");
+        Entity entity = new HttpFetcher().fetch(link).getEntity();
+        Parser parser = new ClassPathXmlApplicationContext("test-parser.impl.NekoHtmlParserTest.xml").getBean("nekoHtmlParser0", NekoHtmlParser.class);
+        Map<String, Map<String, Set<Link>>> result = (Map<String, Map<String, Set<Link>>>) parser.parse(link, entity, null).getData();
+        assertNotNull(result);
+        assertEquals(1, result.size());
+        assertEquals("A", result.keySet().iterator().next());
+        assertEquals(1, result.get("A").size());
+        assertEquals("href", result.get("A").keySet().iterator().next());
+        assertTrue(result.get("A").get("href").size() > 100);
+    }
+
+
+    @Test public void testParser_Style() throws URISyntaxException, FetcherException, ParserException{
+        Link link = new Link("http://www.w3schools.com/css/tryit_view.asp?filename=trycss_background-image");
+        Entity entity = new HttpFetcher().fetch(link).getEntity();
+        Parser parser = new ClassPathXmlApplicationContext("test-parser.impl.NekoHtmlParserTest.xml").getBean("nekoHtmlParser1", NekoHtmlParser.class);
+        Map<String, Map<String, Set<Link>>> result = (Map<String, Map<String, Set<Link>>>) parser.parse(link, entity, null).getData();
+        assertNotNull(result);
+        assertEquals(1, result.size());
+        assertEquals("STYLE", result.keySet().iterator().next());
+        assertEquals(1, result.get("STYLE").size());
+        assertEquals("", result.get("STYLE").keySet().iterator().next());
+        assertTrue(result.get("STYLE").get("").size() == 1);
+        assertEquals("body {background-image:url('paper.gif')}", result.get("STYLE").get("").toArray()[0]);
+    }
+}
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java (added)
+++ incubator/droids/trunk/droids-crawler/src/test/java/org/apache/droids/crawler/util/ParamUtilsTest.java Tue Sep  1 22:11:29 2009
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.crawler.util;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ParamUtilsTest{
+
+    @Test public void testResolve(){
+        ApplicationContext context = new ClassPathXmlApplicationContext("test-util.ParamUtilsTest.xml");
+        assertTrue(context.containsBean("map"));
+
+        Map cfg = new HashMap();
+        cfg.put("key", Map.class);
+        Map result = (Map) ParamUtils.resolve(cfg, "key", Map.class, context);
+        assertNotNull(result);
+        assertTrue(result.containsKey("k"));
+        assertEquals("v", result.get("k"));
+    }
+}

Added: incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/log4j.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration debug="false" xmlns:log4j="http://jakarta.apache.org/log4j/">
+    <!-- http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html -->
+
+    <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p [%-40.40c] [%-11t] [%t]  - %m%n"/>
+            <!--<param name="ConversionPattern" value="%d %-5p [%-30.40c{1}] [%t]  - %m%n"/>-->
+        </layout>
+    </appender>
+
+    <logger name="org.apache.http.wire" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.apache.droids.crawler.CrawlerWorker" additivity="false">
+        <level value="trace"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.apache.droids.crawler" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <root>
+        <priority value="info"/>
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+</log4j:configuration>

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-LocalCrawlerServiceTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+    <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+
+    <import resource="classpath:/test-fetcher.HttpFetcherTest.xml"/>
+
+    <bean class="org.apache.droids.crawler.filter.fetch.LinkAttributeFilter"/>
+
+    <bean id="nekoHtmlParser" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+
+    <bean id="crawlerService" class="org.apache.droids.crawler.LocalCrawlerService">
+        <property name="fetcherFactory">
+            <bean class="org.apache.droids.crawler.fetcher.DefaultFetcherFactory">
+                <property name="fetchers">
+                    <util:list>
+                        <bean class="org.apache.droids.crawler.fetcher.HttpFetcher"/>
+                    </util:list>
+                </property>
+            </bean>
+        </property>
+        <property name="parserFactory">
+            <bean class="org.apache.droids.crawler.parser.DefaultParserFactory">
+                <property name="parsers">
+                    <util:list>
+                        <bean class="org.apache.droids.crawler.parser.impl.NekoHtmlParser"/>
+                    </util:list>
+                </property>
+            </bean>
+        </property>
+        <property name="extractors">
+            <util:list>
+                <bean class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+            </util:list>
+        </property>
+    </bean>
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-RoundRobinCrawlerServiceTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+
+    <bean id="crawlerService" class="org.apache.droids.crawler.RoundRobinCrawlerService">
+        <property name="crawlerServices">
+            <util:list>
+                <bean id="service0" class="org.apache.droids.crawler.DummyCrawlerService"/>
+                <bean id="service1" class="org.apache.droids.crawler.DummyCrawlerService"/>
+                <bean id="service2" class="org.apache.droids.crawler.DummyCrawlerService"/>
+            </util:list>
+        </property>
+    </bean>
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-StandaloneCrawlerControllerTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+    <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+    <bean id="queue" class="org.apache.droids.crawler.util.DefaultLinkQueue"/>
+
+    <bean id="worker" class="org.apache.droids.crawler.CrawlerWorker" scope="prototype">
+        <property name="queue" ref="queue"/>
+        <property name="crawlerService" ref="crawlerService"/>
+    </bean>
+
+    <bean id="crawlerController" class="org.apache.droids.crawler.StandaloneCrawlerController" init-method="init">
+        <property name="executorService">
+            <bean class="org.apache.droids.crawler.util.CrawlerExecutorService">
+                <constructor-arg index="0" value="1"/>
+                <constructor-arg index="1">
+                    <bean class="org.apache.droids.core.thread.DefaultThreadFactory"/>
+                </constructor-arg>
+                <constructor-arg index="2">
+                    <bean class="org.apache.droids.core.thread.DefaultRejectedExecutionHandler"/>
+                </constructor-arg>
+            </bean>
+        </property>
+        <property name="seeds">
+            <util:list>
+                <value>http://incubator.apache.org/droids/</value>
+            </util:list>
+        </property>
+        <property name="queue" ref="queue"/>
+    </bean>
+
+
+    <bean id="httpFetcher" class="org.apache.droids.crawler.fetcher.HttpFetcher" scope="prototype"/>
+    <bean id="htmlParser" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+    <bean id="htmlExtractor" class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+
+    <bean id="crawlerService" class="org.apache.droids.crawler.LocalCrawlerService">
+        <property name="fetcherFactory">
+            <bean class="org.apache.droids.crawler.fetcher.DefaultFetcherFactory">
+                <property name="fetchers">
+                    <util:list>
+                        <ref bean="httpFetcher"/>
+                    </util:list>
+                </property>
+            </bean>
+        </property>
+        <property name="parserFactory">
+            <bean class="org.apache.droids.crawler.parser.DefaultParserFactory">
+                <property name="parsers">
+                    <util:list>
+                        <ref bean="htmlParser"/>
+                    </util:list>
+                </property>
+            </bean>
+        </property>
+        <property name="extractors">
+            <util:list>
+                <ref bean="htmlExtractor"/>
+            </util:list>
+        </property>
+    </bean>
+
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-extractor.HtmlElementLinkExtractorTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <import resource="classpath:/test-parser.impl.NekoHtmlParserTest.xml"/>
+
+    <bean id="htmlElementLinkExtractor" class="org.apache.droids.crawler.extractor.HtmlElementLinkExtractor"/>
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-fetcher.HttpFetcherTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <!--<context:annotation-config/>-->
+    <!--<context:component-scan base-package="org.apache.droids.crawler"/>-->
+
+    <!-- this is an example of how to configure HttpClient in Spring -->
+    <bean id="httpParams" class="org.apache.http.params.BasicHttpParams"/>
+    <bean class="org.apache.http.params.HttpConnectionParamBean">
+        <constructor-arg index="0">
+            <ref bean="httpParams"/>
+        </constructor-arg>
+        <property name="connectionTimeout" value="60000"/>
+        <property name="tcpNoDelay" value="false"/>
+        <property name="staleCheckingEnabled" value="false"/>
+        <property name="socketBufferSize" value="8192"/>
+    </bean>
+    <bean class="org.apache.http.params.HttpProtocolParamBean">
+        <constructor-arg index="0">
+            <ref bean="httpParams"/>
+        </constructor-arg>
+        <property name="contentCharset" value="ISO-8859-1"/>
+        <!--<property name="httpElementCharset" value="UTF-8"/>-->
+        <property name="useExpectContinue" value="true"/>
+        <property name="userAgent"
+                  value="Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.1) Gecko/20090715 Firefox/3.0.10"/>
+        <!--<property name="version" value="1.1"/>-->
+    </bean>
+
+
+    <bean id="httpClient" class="org.apache.http.impl.client.DefaultHttpClient">
+        <constructor-arg index="0" ref="httpParams"/>
+    </bean>
+
+    <bean id="httpFetcher" class="org.apache.droids.crawler.fetcher.HttpFetcher" scope="prototype">
+        <property name="httpClient" ref="httpClient"/>
+    </bean>
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.DefaultParserFactoryTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+    <context:component-scan base-package="org.apache.droids.crawler"/>
+
+    <!-- no configuration of parsers to test autowiring -->
+    <bean id="parserFactory" class="org.apache.droids.crawler.parser.DefaultParserFactory"/>
+    <bean class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype"/>
+    <!--<bean class="org.apache.droids.crawler.parser.impl.JerichoHtmlParser" scope="prototype"/>-->
+    <bean class="org.apache.droids.crawler.parser.impl.SAXElementParser" scope="prototype"/>
+    <bean class="org.apache.droids.crawler.parser.impl.StAXElementParser" scope="prototype"/>
+
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-parser.impl.NekoHtmlParserTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <context:annotation-config/>
+    <context:component-scan base-package="org.apache.droids.crawler"/>
+
+    <!-- no configuration of parsers to test autowiring -->
+    <bean id="nekoHtmlParser0" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype">
+        <property name="elements">
+            <util:map>
+                <entry key="A">
+                    <util:list>
+                        <value>href</value>
+                    </util:list>
+                </entry>
+            </util:map>
+        </property>
+    </bean>
+
+    <!-- no configuration of parsers to test autowiring -->
+    <bean id="nekoHtmlParser1" class="org.apache.droids.crawler.parser.impl.NekoHtmlParser" scope="prototype">
+        <property name="elements">
+            <util:map>
+                <entry key="STYLE">
+                    <util:list>
+                        <value></value>
+                    </util:list>
+                </entry>
+            </util:map>
+        </property>
+    </bean>
+
+</beans>
\ No newline at end of file

Added: incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml
URL: http://svn.apache.org/viewvc/incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml?rev=810273&view=auto
==============================================================================
--- incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml (added)
+++ incubator/droids/trunk/droids-crawler/src/test/resources/test-util.ParamUtilsTest.xml Tue Sep  1 22:11:29 2009
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+        http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+        http://www.springframework.org/schema/context
+        http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+        http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+    <util:map id="map">
+        <entry key="k" value="v"/>
+    </util:map>
+
+</beans>
\ No newline at end of file



Mime
View raw message